Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Replace Ingest ScriptContext with Custom Interface #32003

Merged
merged 9 commits into from
Jul 13, 2018
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.Processor;
import org.elasticsearch.script.ExecutableScript;
import org.elasticsearch.script.IngestScript;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptException;
import org.elasticsearch.script.ScriptService;
Expand Down Expand Up @@ -73,10 +73,8 @@ public final class ScriptProcessor extends AbstractProcessor {
*/
@Override
public void execute(IngestDocument document) {
ExecutableScript.Factory factory = scriptService.compile(script, ExecutableScript.INGEST_CONTEXT);
ExecutableScript executableScript = factory.newInstance(script.getParams());
executableScript.setNextVar("ctx", document.getSourceAndMetadata());
executableScript.run();
IngestScript.Factory factory = scriptService.compile(script, IngestScript.CONTEXT);
factory.newInstance(script.getParams()).execute(document.getSourceAndMetadata());
}

@Override
Expand Down Expand Up @@ -108,7 +106,7 @@ public ScriptProcessor create(Map<String, Processor.Factory> registry, String pr

// verify script is able to be compiled before successfully creating processor.
try {
scriptService.compile(script, ExecutableScript.INGEST_CONTEXT);
scriptService.compile(script, IngestScript.CONTEXT);
} catch (ScriptException e) {
throw newConfigurationException(TYPE, processorTag, null, e);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,14 @@

import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.RandomDocumentPicks;
import org.elasticsearch.script.ExecutableScript;
import org.elasticsearch.script.IngestScript;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.test.ESTestCase;

import static org.hamcrest.Matchers.hasKey;
import static org.hamcrest.core.Is.is;
import static org.mockito.Matchers.anyMapOf;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
Expand All @@ -45,9 +46,9 @@ public void testScripting() throws Exception {

ScriptService scriptService = mock(ScriptService.class);
Script script = mockScript("_script");
ExecutableScript.Factory factory = mock(ExecutableScript.Factory.class);
ExecutableScript executableScript = mock(ExecutableScript.class);
when(scriptService.compile(script, ExecutableScript.INGEST_CONTEXT)).thenReturn(factory);
IngestScript.Factory factory = mock(IngestScript.Factory.class);
IngestScript executableScript = mock(IngestScript.class);
when(scriptService.compile(script, IngestScript.CONTEXT)).thenReturn(factory);
when(factory.newInstance(any())).thenReturn(executableScript);

Map<String, Object> document = new HashMap<>();
Expand All @@ -58,7 +59,7 @@ public void testScripting() throws Exception {
doAnswer(invocationOnMock -> {
ingestDocument.setFieldValue("bytes_total", randomBytesTotal);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

one thing we do not check here is that the Map passed in as the ctx is actually

IngestDocument#getSourceAndMetadata()

Do you think that would be useful here?

We do have checks in rest-api-spec/test/ingest/50_script_processor_using_painless.yml that we are actually modifying the document, so that may be sufficient. Seems like it would be nice to validate this from unit tests too

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@talevy makes sense + is trivial to do => sec adjusting :)

Copy link
Member Author

@original-brownbear original-brownbear Jul 12, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@talevy done in 89874c9 :)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

cooool. thanks!

return null;
}).when(executableScript).run();
}).when(executableScript).execute(anyMapOf(String.class, Object.class));

ScriptProcessor processor = new ScriptProcessor(randomAlphaOfLength(10), script, scriptService);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,5 +50,4 @@ interface Factory {
// TODO: remove these once each has its own script interface
ScriptContext<Factory> AGGS_CONTEXT = new ScriptContext<>("aggs_executable", Factory.class);
ScriptContext<Factory> UPDATE_CONTEXT = new ScriptContext<>("update", Factory.class);
ScriptContext<Factory> INGEST_CONTEXT = new ScriptContext<>("ingest", Factory.class);
}
49 changes: 49 additions & 0 deletions server/src/main/java/org/elasticsearch/script/IngestScript.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@

/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.script;

import java.util.Map;

public abstract class IngestScript {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe add class level java docs here?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sounds good, added in 762d2f7 :)


public static final String[] PARAMETERS = { "ctx" };

/** The context used to compile {@link IngestScript} factories. */
public static final ScriptContext<Factory> CONTEXT = new ScriptContext<>("ingest", Factory.class);

/** The generic runtime parameters for the script. */
private final Map<String, Object> params;

public IngestScript(Map<String, Object> params) {
this.params = params;
}

/** Return the parameters for this script. */
public Map<String, Object> getParams() {
return params;
}

public abstract void execute(Map<String, Object> ctx);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

While this will be needed for backcompat in 6.x, I think we want to expose IngestDocument as a variable directly here. But this can be done as a followup (along with deprecation control of the old ctx variable).


public interface Factory {
IngestScript newInstance(Map<String, Object> params);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ public class ScriptModule {
ExecutableScript.CONTEXT,
ExecutableScript.AGGS_CONTEXT,
ExecutableScript.UPDATE_CONTEXT,
ExecutableScript.INGEST_CONTEXT,
IngestScript.CONTEXT,
FilterScript.CONTEXT,
SimilarityScript.CONTEXT,
SimilarityWeightScript.CONTEXT,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ public void testAllowAllScriptContextSettings() throws IOException {
assertCompileAccepted("painless", "script", ScriptType.INLINE, SearchScript.CONTEXT);
assertCompileAccepted("painless", "script", ScriptType.INLINE, SearchScript.AGGS_CONTEXT);
assertCompileAccepted("painless", "script", ScriptType.INLINE, ExecutableScript.UPDATE_CONTEXT);
assertCompileAccepted("painless", "script", ScriptType.INLINE, ExecutableScript.INGEST_CONTEXT);
assertCompileAccepted("painless", "script", ScriptType.INLINE, IngestScript.CONTEXT);
}

public void testAllowSomeScriptTypeSettings() throws IOException {
Expand Down Expand Up @@ -209,13 +209,13 @@ public void testAllowNoScriptContextSettings() throws IOException {
}

public void testCompileNonRegisteredContext() throws IOException {
contexts.remove(ExecutableScript.INGEST_CONTEXT.name);
contexts.remove(IngestScript.CONTEXT.name);
buildScriptService(Settings.EMPTY);

String type = scriptEngine.getType();
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
scriptService.compile(new Script(ScriptType.INLINE, type, "test", Collections.emptyMap()), ExecutableScript.INGEST_CONTEXT));
assertThat(e.getMessage(), containsString("script context [" + ExecutableScript.INGEST_CONTEXT.name + "] not supported"));
scriptService.compile(new Script(ScriptType.INLINE, type, "test", Collections.emptyMap()), IngestScript.CONTEXT));
assertThat(e.getMessage(), containsString("script context [" + IngestScript.CONTEXT.name + "] not supported"));
}

public void testCompileCountedInCompilationStats() throws IOException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,16 @@ public <T> T compile(String name, String source, ScriptContext<T> context, Map<S
} else if (context.instanceClazz.equals(ExecutableScript.class)) {
ExecutableScript.Factory factory = mockCompiled::createExecutableScript;
return context.factoryClazz.cast(factory);
} else if (context.instanceClazz.equals(IngestScript.class)) {
IngestScript.Factory factory = parameters -> new IngestScript(parameters) {
@Override
public void execute(Map<String, Object> ctx) {
if (script != null) {
script.apply(ctx);
}
}
};
return context.factoryClazz.cast(factory);
} else if (context.instanceClazz.equals(TemplateScript.class)) {
TemplateScript.Factory factory = vars -> {
// TODO: need a better way to implement all these new contexts
Expand Down