From 8ce09b3f6b9b7244712520dcc89f2b4c19a38549 Mon Sep 17 00:00:00 2001 From: ssambasu Date: Fri, 28 Dec 2018 17:39:19 -0800 Subject: [PATCH 1/2] Passing more than one options in input flows using mlcp --- .../4/transforms/mlcp-flow-transform.sjs | 22 ++++++++++--- .../4/transforms/mlcp-flow-transform.xqy | 32 ++++++++++++++----- .../hub_integration/EndToEndFlowTests.java | 3 +- 3 files changed, 43 insertions(+), 14 deletions(-) diff --git a/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/4/transforms/mlcp-flow-transform.sjs b/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/4/transforms/mlcp-flow-transform.sjs index db9269ae3e..a33d052684 100644 --- a/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/4/transforms/mlcp-flow-transform.sjs +++ b/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/4/transforms/mlcp-flow-transform.sjs @@ -23,7 +23,20 @@ const tracelib = require("/data-hub/4/impl/trace-lib.sjs"); function transform(content, context) { let uri = content.uri; let params = {}; - let splits = context.transform_param.split(','); + let optionsString = null; + let parsedTransformParam = null; + let transformString = context.transform_param; + let pattern = '^.*(options=\{.*\}).*$'; + let match = new RegExp(pattern).exec(transformString); + if (match === null){ + parsedTransformParam = transformString; + } + else{ + optionsString = match[1]; + parsedTransformParam = transformString.replace(optionsString, ''); + } + + let splits = parsedTransformParam.split(','); for (let i in splits) { let pair = splits[i]; let parts = pair.split('='); @@ -38,11 +51,10 @@ function transform(content, context) { if (!flow) { fn.error(null, "RESTAPI-SRVEXERR", "The specified flow " + params.flow + " is missing."); } - - // configure the options let options = {}; - if (params.options) { - options = JSON.parse(params.options); + if (optionsString) { + let splits = optionsString.split("="); + options = JSON.parse(splits[1]); } flowlib.setDefaultOptions(options, flow); diff --git a/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/4/transforms/mlcp-flow-transform.xqy b/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/4/transforms/mlcp-flow-transform.xqy index 3149583489..1102417058 100644 --- a/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/4/transforms/mlcp-flow-transform.xqy +++ b/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/4/transforms/mlcp-flow-transform.xqy @@ -52,13 +52,23 @@ declare function mlcpFlow:transform( let $uri := map:get($content, "uri") return perf:log('mlcp-flow-transform(' || $uri || ')', function() { - let $params := map:new(( - for $pair in map:get($context, 'transform_param') ! fn:tokenize(., ",") - let $parts := fn:tokenize($pair, "=") - return - map:entry($parts[1], $parts[2]) - )) - + let $transform-string := map:get($context, 'transform_param') + let $options-string := replace($transform-string, '^.*(options=\{.*\}).*$', '$1') + let $parsed-transform-string := + if ($transform-string = $options-string) then + $transform-string + else + fn:replace($transform-string, 'options=\{.*\}', '') + let $params := map:new( + for $pair in $parsed-transform-string ! fn:tokenize(., ",") + let $parts := fn:tokenize($pair, "=") + return + if(fn:not(fn:empty($parts[1]))) then + map:entry($parts[1], $parts[2]) + else + () + ) + let $job-id := (map:get($params, "job-id"), sem:uuid-string())[1] let $entity-name := map:get($params, 'entity-name') ! xdmp:url-decode(.) let $flow-name := map:get($params, 'flow-name') ! xdmp:url-decode(.) @@ -74,8 +84,14 @@ declare function mlcpFlow:transform( fn:error((), "RESTAPI-SRVEXERR", "The specified flow " || map:get($params, "flow") || " is missing.") (: configure the options :) + let $opts := map:new( + let $opt-parts := fn:tokenize($options-string, "=") + + return + map:entry($opt-parts[1], $opt-parts[2]) + ) let $options as map:map := ( - map:get($params, "options") ! xdmp:unquote(.)/object-node(), + map:get($opts, "options") ! xdmp:unquote(.)/object-node(), map:map() )[1] let $_ := flow:set-default-options($options, $flow) diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub_integration/EndToEndFlowTests.java b/marklogic-data-hub/src/test/java/com/marklogic/hub_integration/EndToEndFlowTests.java index f296634d10..fe8d24cc8f 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub_integration/EndToEndFlowTests.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub_integration/EndToEndFlowTests.java @@ -140,7 +140,7 @@ public void setupEach() { clearDatabases(HubConfig.DEFAULT_STAGING_NAME, HubConfig.DEFAULT_FINAL_NAME, HubConfig.DEFAULT_JOB_NAME); enableTracing(); - enableDebugging(); + disableDebugging(); flowRunnerDataMovementManager = flowRunnerClient.newDataMovementManager(); @@ -356,6 +356,7 @@ public List generateExtraPluginTests() { tests.add(DynamicTest.dynamicTest(flowName + " MLCP", () -> { Map options = new HashMap<>(); options.put("extraPlugin", true); + options.put("secondOption", "secondValue"); FinalCounts finalCounts = new FinalCounts(1, 0, 1, 1, 0, 0, 1, 0, 0, 0, "FINISHED"); testInputFlowViaMlcp(prefix, useEs ? "-es" : "", flowRunnerClient, codeFormat, dataFormat, useEs, options, finalCounts); })); From a64994a347273eb9b51508e9ca3330071ae0ed5a Mon Sep 17 00:00:00 2001 From: ssambasu Date: Fri, 28 Dec 2018 17:42:38 -0800 Subject: [PATCH 2/2] Enables debugging --- .../java/com/marklogic/hub_integration/EndToEndFlowTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub_integration/EndToEndFlowTests.java b/marklogic-data-hub/src/test/java/com/marklogic/hub_integration/EndToEndFlowTests.java index fe8d24cc8f..da6ffbcadb 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub_integration/EndToEndFlowTests.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub_integration/EndToEndFlowTests.java @@ -140,7 +140,7 @@ public void setupEach() { clearDatabases(HubConfig.DEFAULT_STAGING_NAME, HubConfig.DEFAULT_FINAL_NAME, HubConfig.DEFAULT_JOB_NAME); enableTracing(); - disableDebugging(); + enableDebugging(); flowRunnerDataMovementManager = flowRunnerClient.newDataMovementManager();