From 3a72661a9e0677b9afde538a9b7c1de191772965 Mon Sep 17 00:00:00 2001 From: Noah Cover Date: Wed, 9 Apr 2025 16:54:26 -0700 Subject: [PATCH 1/2] NIFI-14459: Creating processor to update box metadata template --- .../box/UpdateBoxMetadataTemplate.java | 485 ++++++++++++++++++ .../box/UpdateBoxMetadataTemplateTest.java | 262 ++++++++++ 2 files changed, 747 insertions(+) create mode 100644 nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/main/java/org/apache/nifi/processors/box/UpdateBoxMetadataTemplate.java create mode 100644 nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/test/java/org/apache/nifi/processors/box/UpdateBoxMetadataTemplateTest.java diff --git a/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/main/java/org/apache/nifi/processors/box/UpdateBoxMetadataTemplate.java b/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/main/java/org/apache/nifi/processors/box/UpdateBoxMetadataTemplate.java new file mode 100644 index 000000000000..5b7ab42484c5 --- /dev/null +++ b/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/main/java/org/apache/nifi/processors/box/UpdateBoxMetadataTemplate.java @@ -0,0 +1,485 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.box; + +import com.box.sdk.BoxAPIConnection; +import com.box.sdk.BoxAPIResponseException; +import com.box.sdk.MetadataTemplate; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.SeeAlso; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.box.controllerservices.BoxClientService; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.expression.ExpressionLanguageScope; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.serialization.RecordReader; +import org.apache.nifi.serialization.RecordReaderFactory; +import org.apache.nifi.serialization.record.Record; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static java.lang.String.valueOf; +import static org.apache.nifi.processors.box.BoxFileAttributes.ERROR_CODE; +import static org.apache.nifi.processors.box.BoxFileAttributes.ERROR_CODE_DESC; +import static org.apache.nifi.processors.box.BoxFileAttributes.ERROR_MESSAGE; +import static org.apache.nifi.processors.box.BoxFileAttributes.ERROR_MESSAGE_DESC; +import static org.apache.nifi.processors.box.CreateBoxMetadataTemplate.SCOPE_ENTERPRISE; + +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@Tags({"box", "storage", "metadata", "templates", "update"}) +@CapabilityDescription(""" + Updates a Box metadata template using the desired schema from the flowFile content.\s + Takes in the desired end state of the template, compares it with the existing template, + and computes the necessary operations to transform the template to the desired state. + Admin permissions are required to update templates. + """) +@SeeAlso({ListBoxFileMetadataTemplates.class, CreateBoxMetadataTemplate.class, UpdateBoxFileMetadataInstance.class}) +@WritesAttributes({ + @WritesAttribute(attribute = "box.template.key", description = "The template key that was updated"), + @WritesAttribute(attribute = "box.template.scope", description = "The template scope"), + @WritesAttribute(attribute = "box.template.operations.count", description = "Number of operations performed on the template"), + @WritesAttribute(attribute = ERROR_CODE, description = ERROR_CODE_DESC), + @WritesAttribute(attribute = ERROR_MESSAGE, description = ERROR_MESSAGE_DESC) +}) +public class UpdateBoxMetadataTemplate extends AbstractProcessor { + + private static final Set VALID_FIELD_TYPES = Set.of("string", "float", "date", "enum", "multiSelect"); + + public static final PropertyDescriptor TEMPLATE_KEY = new PropertyDescriptor.Builder() + .name("Template Key") + .description("The key of the metadata template to update.") + .required(true) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + public static final PropertyDescriptor SCOPE = new PropertyDescriptor.Builder() + .name("Scope") + .description("The scope of the metadata template. Usually 'enterprise'.") + .required(true) + .defaultValue(SCOPE_ENTERPRISE) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + public static final PropertyDescriptor RECORD_READER = new PropertyDescriptor.Builder() + .name("Record Reader") + .description("The Record Reader to use for parsing the incoming data with the desired template schema") + .required(true) + .identifiesControllerService(RecordReaderFactory.class) + .build(); + + public static final Relationship REL_SUCCESS = new Relationship.Builder() + .name("success") + .description("A FlowFile is routed to this relationship after a template has been successfully updated.") + .build(); + + public static final Relationship REL_FAILURE = new Relationship.Builder() + .name("failure") + .description("A FlowFile is routed to this relationship if an error occurs during template update.") + .build(); + + public static final Relationship REL_TEMPLATE_NOT_FOUND = new Relationship.Builder() + .name("template not found") + .description("FlowFiles for which the specified metadata template was not found will be routed to this relationship.") + .build(); + + private static final Set RELATIONSHIPS = Set.of( + REL_SUCCESS, + REL_FAILURE, + REL_TEMPLATE_NOT_FOUND + ); + + private static final List PROPERTY_DESCRIPTORS = List.of( + BoxClientService.BOX_CLIENT_SERVICE, + TEMPLATE_KEY, + SCOPE, + RECORD_READER + ); + + private volatile BoxAPIConnection boxAPIConnection; + + @Override + protected List getSupportedPropertyDescriptors() { + return PROPERTY_DESCRIPTORS; + } + + @Override + public Set getRelationships() { + return RELATIONSHIPS; + } + + @OnScheduled + public void onScheduled(final ProcessContext context) { + boxAPIConnection = getBoxAPIConnection(context); + } + + protected BoxAPIConnection getBoxAPIConnection(final ProcessContext context) { + final BoxClientService boxClientService = context.getProperty(BoxClientService.BOX_CLIENT_SERVICE) + .asControllerService(BoxClientService.class); + return boxClientService.getBoxApiConnection(); + } + + @Override + public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { + FlowFile flowFile = session.get(); + if (flowFile == null) { + return; + } + + final String templateKey = context.getProperty(TEMPLATE_KEY).evaluateAttributeExpressions(flowFile).getValue(); + final String scope = context.getProperty(SCOPE).evaluateAttributeExpressions(flowFile).getValue(); + final RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER).asControllerService(RecordReaderFactory.class); + + try { + // Get the current template + final MetadataTemplate existingTemplate = getMetadataTemplate(scope, templateKey); + + // Parse the desired state from the flowFile + final List desiredFields = readDesiredFields(session, flowFile, recordReaderFactory); + + if (desiredFields.isEmpty()) { + flowFile = session.putAttribute(flowFile, "box.error.message", "No valid metadata field specifications found in the input"); + session.transfer(flowFile, REL_FAILURE); + return; + } + + // Generate operations to transform existing template to desired state + final List operations = generateOperations(existingTemplate, desiredFields); + + if (!operations.isEmpty()) { + getLogger().info("Updating metadata template {} with {} operations", templateKey, operations.size()); + updateMetadataTemplate(scope, templateKey, operations); + } + + final Map attributes = Map.of( + "box.template.key", templateKey, + "box.template.scope", scope, + "box.template.operations.count", String.valueOf(operations.size())); + flowFile = session.putAllAttributes(flowFile, attributes); + + session.getProvenanceReporter().modifyAttributes(flowFile, "Updated Box metadata template: " + templateKey); + session.transfer(flowFile, REL_SUCCESS); + + } catch (final BoxAPIResponseException e) { + flowFile = session.putAttribute(flowFile, ERROR_CODE, valueOf(e.getResponseCode())); + flowFile = session.putAttribute(flowFile, ERROR_MESSAGE, e.getMessage()); + + if (e.getResponseCode() == 404) { + getLogger().warn("Box metadata template with key {} in scope {} was not found", templateKey, scope); + session.transfer(flowFile, REL_TEMPLATE_NOT_FOUND); + } else { + getLogger().error("Couldn't update metadata template with key [{}]", templateKey, e); + session.transfer(flowFile, REL_FAILURE); + } + } catch (final Exception e) { + getLogger().error("Error processing metadata template update", e); + flowFile = session.putAttribute(flowFile, ERROR_MESSAGE, e.getMessage()); + session.transfer(flowFile, REL_FAILURE); + } + } + + private List readDesiredFields(final ProcessSession session, + final FlowFile flowFile, + final RecordReaderFactory recordReaderFactory) throws Exception { + final List fields = new ArrayList<>(); + final Set processedKeys = new HashSet<>(); + final List errors = new ArrayList<>(); + + try (final InputStream inputStream = session.read(flowFile); + final RecordReader recordReader = recordReaderFactory.createRecordReader(flowFile, inputStream, getLogger())) { + + Record record; + while ((record = recordReader.nextRecord()) != null) { + processFieldRecord(record, fields, processedKeys, errors); + } + } + + if (!errors.isEmpty()) { + String errorMessage = "Error parsing field definitions: " + String.join(", ", errors); + throw new ProcessException(errorMessage); + } + + return fields; + } + + private void processFieldRecord(final Record record, + final List fields, + final Set processedKeys, + final List errors) { + // Extract and validate key (required) + final Object keyObj = record.getValue("key"); + if (keyObj == null) { + errors.add("Record is missing a key field"); + return; + } + final String key = keyObj.toString(); + + if (processedKeys.contains(key)) { + errors.add("Duplicate key '" + key + "' found in record"); + return; + } + + // Extract and validate type (required) + final Object typeObj = record.getValue("type"); + if (typeObj == null) { + errors.add("Record with key '" + key + "' is missing a type field"); + return; + } + final String type = typeObj.toString().toLowerCase(); + + if (!VALID_FIELD_TYPES.contains(type)) { + errors.add("Record with key '" + key + "' has an invalid type: '" + type + + "'. Valid types are: " + String.join(", ", VALID_FIELD_TYPES)); + return; + } + + final FieldDefinition field = new FieldDefinition(); + field.key = key; + field.type = type; + + final Object displayNameObj = record.getValue("displayName"); + if (displayNameObj != null) { + field.displayName = displayNameObj.toString(); + } + + final Object hiddenObj = record.getValue("hidden"); + if (hiddenObj != null) { + field.hidden = Boolean.parseBoolean(hiddenObj.toString()); + } + + final Object descriptionObj = record.getValue("description"); + if (descriptionObj != null) { + field.description = descriptionObj.toString(); + } + + if ("enum".equals(type) || "multiSelect".equals(type)) { + final Object optionsObj = record.getValue("options"); + if (optionsObj instanceof List optionsList) { + field.options = optionsList.stream() + .filter(obj -> obj != null) + .map(Object::toString) + .collect(Collectors.toList()); + } + } + + fields.add(field); + processedKeys.add(key); + } + + private List generateOperations(final MetadataTemplate existingTemplate, + final List desiredFields) { + final List operations = new ArrayList<>(); + final Map existingFieldsByKey = new HashMap<>(); + + // Create a map of existing fields by key for efficient lookup + for (MetadataTemplate.Field field : existingTemplate.getFields()) { + existingFieldsByKey.put(field.getKey(), field); + } + + // Process each desired field + for (FieldDefinition desiredField : desiredFields) { + MetadataTemplate.Field existingField = existingFieldsByKey.get(desiredField.key); + + if (existingField == null) { + // Field doesn't exist - add it + operations.add(createAddFieldOperation(desiredField)); + } else { + // Field exists - check if it needs updating + Map changes = getFieldChanges(existingField, desiredField); + if (!changes.isEmpty()) { + operations.add(createEditFieldOperation(existingField.getKey(), changes)); + } + + // Remove processed field from the map so we can track which fields to remove + existingFieldsByKey.remove(desiredField.key); + } + } + + // Any remaining fields in existingFieldsByKey are not in the desired state - remove them + for (String keyToRemove : existingFieldsByKey.keySet()) { + operations.add(createRemoveFieldOperation(keyToRemove)); + } + + return operations; + } + + private Map getFieldChanges(final MetadataTemplate.Field existingField, + final FieldDefinition desiredField) { + final Map changes = new HashMap<>(); + + // Check if key has changed + if (!existingField.getKey().equals(desiredField.key)) { + changes.put("key", desiredField.key); + } + + // Check if displayName has changed + if (desiredField.displayName != null && + (existingField.getDisplayName() == null || !existingField.getDisplayName().equals(desiredField.displayName))) { + changes.put("displayName", desiredField.displayName); + } + + // Check if type has changed (this is a rare case) + if (!existingField.getType().equals(desiredField.type)) { + changes.put("type", desiredField.type); + } + + // Check if hidden state has changed + if (desiredField.hidden != existingField.getIsHidden()) { + changes.put("hidden", desiredField.hidden); + } + + // Check if description has changed + if (desiredField.description != null && + (existingField.getDescription() == null || !existingField.getDescription().equals(desiredField.description))) { + changes.put("description", desiredField.description); + } + + // For enum and multiSelect fields, check if options have changed + if (("enum".equals(desiredField.type) || "multiSelect".equals(desiredField.type)) && + desiredField.options != null && !desiredField.options.isEmpty()) { + + List existingOptions = existingField.getOptions(); + if (existingOptions == null || !new HashSet<>(existingOptions).equals(new HashSet<>(desiredField.options))) { + changes.put("options", desiredField.options); + } + } + + return changes; + } + + private MetadataTemplate.FieldOperation createAddFieldOperation(final FieldDefinition field) { + final StringBuilder jsonBuilder = new StringBuilder(); + jsonBuilder.append("{\"op\":\"addField\",\"data\":{"); + + // Add mandatory fields + jsonBuilder.append("\"key\":\"").append(field.key).append("\","); + jsonBuilder.append("\"type\":\"").append(field.type).append("\","); + + // Add optional fields + if (field.displayName != null) { + jsonBuilder.append("\"displayName\":\"").append(field.displayName).append("\","); + } + + jsonBuilder.append("\"hidden\":").append(field.hidden); + + if (field.description != null) { + jsonBuilder.append(",\"description\":\"").append(field.description).append("\""); + } + + // Add options for enum or multiSelect fields + if (("enum".equals(field.type) || "multiSelect".equals(field.type)) && + field.options != null && !field.options.isEmpty()) { + jsonBuilder.append(",\"options\":["); + + for (int i = 0; i < field.options.size(); i++) { + jsonBuilder.append("{\"key\":\"").append(field.options.get(i)).append("\"}"); + if (i < field.options.size() - 1) { + jsonBuilder.append(","); + } + } + + jsonBuilder.append("]"); + } + + jsonBuilder.append("}}"); + + return new MetadataTemplate.FieldOperation(jsonBuilder.toString()); + } + + private MetadataTemplate.FieldOperation createEditFieldOperation(final String fieldKey, final Map changes) { + final StringBuilder jsonBuilder = new StringBuilder(); + jsonBuilder.append("{\"op\":\"editField\",\"fieldKey\":\"").append(fieldKey).append("\",\"data\":{"); + + int i = 0; + for (Map.Entry entry : changes.entrySet()) { + if (i > 0) { + jsonBuilder.append(","); + } + + jsonBuilder.append("\"").append(entry.getKey()).append("\":"); + + if (entry.getValue() instanceof String) { + jsonBuilder.append("\"").append(entry.getValue()).append("\""); + } else if (entry.getValue() instanceof Boolean) { + jsonBuilder.append(entry.getValue()); + } else if (entry.getValue() instanceof List) { + @SuppressWarnings("unchecked") + List options = (List) entry.getValue(); + jsonBuilder.append("["); + for (int j = 0; j < options.size(); j++) { + jsonBuilder.append("{\"key\":\"").append(options.get(j)).append("\"}"); + if (j < options.size() - 1) { + jsonBuilder.append(","); + } + } + jsonBuilder.append("]"); + } else { + jsonBuilder.append(entry.getValue()); + } + + i++; + } + + jsonBuilder.append("}}"); + + return new MetadataTemplate.FieldOperation(jsonBuilder.toString()); + } + + private MetadataTemplate.FieldOperation createRemoveFieldOperation(final String fieldKey) { + // Create the operation JSON + String removeFieldJson = String.format("{\"op\":\"removeField\",\"fieldKey\":\"%s\"}", fieldKey); + + return new MetadataTemplate.FieldOperation(removeFieldJson); + } + + protected MetadataTemplate getMetadataTemplate(final String scope, final String templateKey) { + return MetadataTemplate.getMetadataTemplate(boxAPIConnection, scope, templateKey); + } + + protected void updateMetadataTemplate(final String scope, + final String templateKey, + final List operations) { + MetadataTemplate.updateMetadataTemplate(boxAPIConnection, scope, templateKey, operations); + } + + private static class FieldDefinition { + String key; + String type; + String displayName; + boolean hidden; + String description; + List options; + } +} diff --git a/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/test/java/org/apache/nifi/processors/box/UpdateBoxMetadataTemplateTest.java b/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/test/java/org/apache/nifi/processors/box/UpdateBoxMetadataTemplateTest.java new file mode 100644 index 000000000000..43b1a9540ef5 --- /dev/null +++ b/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/test/java/org/apache/nifi/processors/box/UpdateBoxMetadataTemplateTest.java @@ -0,0 +1,262 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.box; + +import com.box.sdk.BoxAPIConnection; +import com.box.sdk.BoxAPIResponseException; +import com.box.sdk.MetadataTemplate; +import org.apache.nifi.json.JsonTreeReader; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class UpdateBoxMetadataTemplateTest extends AbstractBoxFileTest { + + private static final String TEMPLATE_KEY = "customerInfo"; + private static final String SCOPE = "enterprise"; + + private List capturedOperations = new ArrayList<>(); + private String capturedScope; + private String capturedTemplateKey; + + private class TestUpdateBoxMetadataTemplate extends UpdateBoxMetadataTemplate { + @Override + protected BoxAPIConnection getBoxAPIConnection(final ProcessContext context) { + return mockBoxAPIConnection; + } + + @Override + protected MetadataTemplate getMetadataTemplate(final String scope, final String templateKey) { + if (scope.equals("notFound") || templateKey.equals("notFound")) { + throw new BoxAPIResponseException("Not Found", 404, "Not Found", null); + } + + final MetadataTemplate mockTemplate = mock(MetadataTemplate.class); + final List fields = new ArrayList<>(); + + // Create an existing field + final MetadataTemplate.Field field1 = mock(MetadataTemplate.Field.class); + when(field1.getKey()).thenReturn("name"); + when(field1.getDisplayName()).thenReturn("Name"); + when(field1.getType()).thenReturn("string"); + when(field1.getIsHidden()).thenReturn(false); + fields.add(field1); + + // Create another existing field + final MetadataTemplate.Field field2 = mock(MetadataTemplate.Field.class); + when(field2.getKey()).thenReturn("industry"); + when(field2.getDisplayName()).thenReturn("Industry"); + when(field2.getType()).thenReturn("enum"); + when(field2.getIsHidden()).thenReturn(false); + when(field2.getOptions()).thenReturn(Arrays.asList("Technology", "Healthcare", "Legal")); + fields.add(field2); + + when(mockTemplate.getFields()).thenReturn(fields); + return mockTemplate; + } + + @Override + protected void updateMetadataTemplate(final String scope, + final String templateKey, + final List operations) { + if (scope.equals("forbidden") || templateKey.equals("forbidden")) { + throw new BoxAPIResponseException("Permission Denied", 403, "Permission Denied", null); + } + + capturedScope = scope; + capturedTemplateKey = templateKey; + capturedOperations = operations; + } + } + + @Override + @BeforeEach + void setUp() throws Exception { + final TestUpdateBoxMetadataTemplate processor = new TestUpdateBoxMetadataTemplate(); + testRunner = TestRunners.newTestRunner(processor); + super.setUp(); + + configureJsonRecordReader(testRunner); + testRunner.setProperty(UpdateBoxMetadataTemplate.TEMPLATE_KEY, TEMPLATE_KEY); + testRunner.setProperty(UpdateBoxMetadataTemplate.SCOPE, SCOPE); + testRunner.setProperty(UpdateBoxMetadataTemplate.RECORD_READER, "json-reader"); + } + + private void configureJsonRecordReader(TestRunner runner) throws InitializationException { + final JsonTreeReader readerService = new JsonTreeReader(); + runner.addControllerService("json-reader", readerService); + runner.enableControllerService(readerService); + } + + @Test + public void testUpdateFields() { + // Content represents desired template state + final String content = """ + [ + { + "key": "company_name", + "displayName": "Company Name", + "type": "string", + "hidden": false + }, + { + "key": "industry", + "displayName": "Industry Sector", + "type": "enum", + "hidden": false, + "options": ["Technology", "Healthcare", "Legal", "Finance"] + }, + { + "key": "tav", + "displayName": "Total Account Value", + "type": "float", + "hidden": false + } + ] + """; + + testRunner.enqueue(content); + testRunner.run(); + + // Verify operations were generated correctly (should create multiple operations) + assertEquals(SCOPE, capturedScope); + assertEquals(TEMPLATE_KEY, capturedTemplateKey); + assertFalse(capturedOperations.isEmpty(), "Should generate operations for template updates"); + + // Check success results + testRunner.assertAllFlowFilesTransferred(UpdateBoxMetadataTemplate.REL_SUCCESS, 1); + final MockFlowFile outFile = testRunner.getFlowFilesForRelationship(UpdateBoxMetadataTemplate.REL_SUCCESS).get(0); + outFile.assertAttributeEquals("box.template.key", TEMPLATE_KEY); + outFile.assertAttributeEquals("box.template.scope", SCOPE); + outFile.assertAttributeEquals("box.template.operations.count", String.valueOf(capturedOperations.size())); + } + + @Test + public void testTemplateNotFound() { + // Setup test to throw not found exception + testRunner.setProperty(UpdateBoxMetadataTemplate.TEMPLATE_KEY, "notFound"); + + final String content = """ + [ + { + "key": "company_name", + "displayName": "Company Name", + "type": "string", + "hidden": false + } + ] + """; + + testRunner.enqueue(content); + testRunner.run(); + + // Verify correct routing + testRunner.assertAllFlowFilesTransferred(UpdateBoxMetadataTemplate.REL_TEMPLATE_NOT_FOUND, 1); + final MockFlowFile outFile = testRunner.getFlowFilesForRelationship(UpdateBoxMetadataTemplate.REL_TEMPLATE_NOT_FOUND).getFirst(); + outFile.assertAttributeExists(BoxFileAttributes.ERROR_MESSAGE); + outFile.assertAttributeEquals(BoxFileAttributes.ERROR_CODE, "404"); + } + + @Test + public void testInvalidInputRecord() { + // Template JSON missing required key field + final String content = """ + [ + { + "displayName": "Company Name", + "type": "string" + } + ] + """; + + testRunner.enqueue(content); + testRunner.run(); + + // Verify correct routing + testRunner.assertAllFlowFilesTransferred(UpdateBoxMetadataTemplate.REL_FAILURE, 1); + } + + @Test + public void testNoChangesNeeded() { + // Input is the same as the current template (no changes needed) + final String content = """ + [ + { + "key": "name", + "displayName": "Name", + "type": "string", + "hidden": false + }, + { + "key": "industry", + "displayName": "Industry", + "type": "enum", + "hidden": false, + "options": ["Technology", "Healthcare", "Legal"] + } + ] + """; + + testRunner.enqueue(content); + testRunner.run(); + + // Verify no operations generated + assertEquals(0, capturedOperations.size(), "Should not generate any operations when no changes needed"); + + // Verify success flow + testRunner.assertAllFlowFilesTransferred(UpdateBoxMetadataTemplate.REL_SUCCESS, 1); + } + + @Test + public void testApiError() { + // Setup test to throw error when updating + testRunner.setProperty(UpdateBoxMetadataTemplate.TEMPLATE_KEY, "forbidden"); + + final String content = """ + [ + { + "key": "company_name", + "displayName": "Company Name", + "type": "string", + "hidden": false + } + ] + """; + + testRunner.enqueue(content); + testRunner.run(); + + // Verify correct routing + testRunner.assertAllFlowFilesTransferred(UpdateBoxMetadataTemplate.REL_FAILURE, 1); + } +} From af76c8b4f7cf588b6f132b59f528eefd2ed4e9b5 Mon Sep 17 00:00:00 2001 From: Noah Cover Date: Wed, 9 Apr 2025 16:54:38 -0700 Subject: [PATCH 2/2] NIFI-14459: Creating processor to update box metadata template --- .../box/UpdateBoxMetadataTemplate.java | 117 +++++++----------- .../org.apache.nifi.processor.Processor | 1 + .../box/UpdateBoxMetadataTemplateTest.java | 22 +--- 3 files changed, 51 insertions(+), 89 deletions(-) diff --git a/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/main/java/org/apache/nifi/processors/box/UpdateBoxMetadataTemplate.java b/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/main/java/org/apache/nifi/processors/box/UpdateBoxMetadataTemplate.java index 5b7ab42484c5..d350e17e384a 100644 --- a/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/main/java/org/apache/nifi/processors/box/UpdateBoxMetadataTemplate.java +++ b/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/main/java/org/apache/nifi/processors/box/UpdateBoxMetadataTemplate.java @@ -46,6 +46,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -59,10 +60,9 @@ @InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) @Tags({"box", "storage", "metadata", "templates", "update"}) @CapabilityDescription(""" - Updates a Box metadata template using the desired schema from the flowFile content.\s - Takes in the desired end state of the template, compares it with the existing template, - and computes the necessary operations to transform the template to the desired state. - Admin permissions are required to update templates. + Updates a Box metadata template using the desired schema from the flowFile content. + Takes in the desired end state of the template, compares it with the existing template,\s + and computes the necessary operations to transform the template to the desired state. """) @SeeAlso({ListBoxFileMetadataTemplates.class, CreateBoxMetadataTemplate.class, UpdateBoxFileMetadataInstance.class}) @WritesAttributes({ @@ -166,9 +166,8 @@ public void onTrigger(final ProcessContext context, final ProcessSession session // Get the current template final MetadataTemplate existingTemplate = getMetadataTemplate(scope, templateKey); - // Parse the desired state from the flowFile + // Parse the desired state from the FlowFile final List desiredFields = readDesiredFields(session, flowFile, recordReaderFactory); - if (desiredFields.isEmpty()) { flowFile = session.putAttribute(flowFile, "box.error.message", "No valid metadata field specifications found in the input"); session.transfer(flowFile, REL_FAILURE); @@ -186,7 +185,8 @@ public void onTrigger(final ProcessContext context, final ProcessSession session final Map attributes = Map.of( "box.template.key", templateKey, "box.template.scope", scope, - "box.template.operations.count", String.valueOf(operations.size())); + "box.template.operations.count", String.valueOf(operations.size()) + ); flowFile = session.putAllAttributes(flowFile, attributes); session.getProvenanceReporter().modifyAttributes(flowFile, "Updated Box metadata template: " + templateKey); @@ -238,6 +238,7 @@ private void processFieldRecord(final Record record, final List fields, final Set processedKeys, final List errors) { + // Extract and validate key (required) final Object keyObj = record.getValue("key"); if (keyObj == null) { @@ -245,7 +246,6 @@ private void processFieldRecord(final Record record, return; } final String key = keyObj.toString(); - if (processedKeys.contains(key)) { errors.add("Duplicate key '" + key + "' found in record"); return; @@ -258,7 +258,6 @@ private void processFieldRecord(final Record record, return; } final String type = typeObj.toString().toLowerCase(); - if (!VALID_FIELD_TYPES.contains(type)) { errors.add("Record with key '" + key + "' has an invalid type: '" + type + "'. Valid types are: " + String.join(", ", VALID_FIELD_TYPES)); @@ -284,11 +283,12 @@ private void processFieldRecord(final Record record, field.description = descriptionObj.toString(); } - if ("enum".equals(type) || "multiSelect".equals(type)) { + // For enum or multiSelect fields, capture options + if (("enum".equals(type) || "multiSelect".equals(type))) { final Object optionsObj = record.getValue("options"); if (optionsObj instanceof List optionsList) { field.options = optionsList.stream() - .filter(obj -> obj != null) + .filter(Objects::nonNull) .map(Object::toString) .collect(Collectors.toList()); } @@ -300,38 +300,32 @@ private void processFieldRecord(final Record record, private List generateOperations(final MetadataTemplate existingTemplate, final List desiredFields) { - final List operations = new ArrayList<>(); - final Map existingFieldsByKey = new HashMap<>(); + final Map existingFieldsByKey = + existingTemplate.getFields().stream() + .collect(Collectors.toMap(MetadataTemplate.Field::getKey, f -> f)); - // Create a map of existing fields by key for efficient lookup - for (MetadataTemplate.Field field : existingTemplate.getFields()) { - existingFieldsByKey.put(field.getKey(), field); - } + final List operations = new ArrayList<>(); - // Process each desired field for (FieldDefinition desiredField : desiredFields) { - MetadataTemplate.Field existingField = existingFieldsByKey.get(desiredField.key); + final MetadataTemplate.Field existingField = existingFieldsByKey.get(desiredField.key); if (existingField == null) { // Field doesn't exist - add it operations.add(createAddFieldOperation(desiredField)); } else { // Field exists - check if it needs updating - Map changes = getFieldChanges(existingField, desiredField); + final Map changes = getFieldChanges(existingField, desiredField); if (!changes.isEmpty()) { operations.add(createEditFieldOperation(existingField.getKey(), changes)); } - - // Remove processed field from the map so we can track which fields to remove existingFieldsByKey.remove(desiredField.key); } } - // Any remaining fields in existingFieldsByKey are not in the desired state - remove them - for (String keyToRemove : existingFieldsByKey.keySet()) { + // Any leftover fields in existingFieldsByKey are not desired - remove them + for (final String keyToRemove : existingFieldsByKey.keySet()) { operations.add(createRemoveFieldOperation(keyToRemove)); } - return operations; } @@ -339,38 +333,32 @@ private Map getFieldChanges(final MetadataTemplate.Field existin final FieldDefinition desiredField) { final Map changes = new HashMap<>(); - // Check if key has changed if (!existingField.getKey().equals(desiredField.key)) { changes.put("key", desiredField.key); } - // Check if displayName has changed - if (desiredField.displayName != null && - (existingField.getDisplayName() == null || !existingField.getDisplayName().equals(desiredField.displayName))) { + if (desiredField.displayName != null && (existingField.getDisplayName() == null + || !existingField.getDisplayName().equals(desiredField.displayName))) { changes.put("displayName", desiredField.displayName); } - // Check if type has changed (this is a rare case) if (!existingField.getType().equals(desiredField.type)) { changes.put("type", desiredField.type); } - // Check if hidden state has changed if (desiredField.hidden != existingField.getIsHidden()) { changes.put("hidden", desiredField.hidden); } - // Check if description has changed - if (desiredField.description != null && - (existingField.getDescription() == null || !existingField.getDescription().equals(desiredField.description))) { + if (desiredField.description != null && (existingField.getDescription() == null + || !existingField.getDescription().equals(desiredField.description))) { changes.put("description", desiredField.description); } - // For enum and multiSelect fields, check if options have changed - if (("enum".equals(desiredField.type) || "multiSelect".equals(desiredField.type)) && - desiredField.options != null && !desiredField.options.isEmpty()) { - - List existingOptions = existingField.getOptions(); + // Check for updated options on enum or multiSelect fields + boolean isEnumOrMultiSelect = "enum".equals(desiredField.type) || "multiSelect".equals(desiredField.type); + if (isEnumOrMultiSelect && desiredField.options != null && !desiredField.options.isEmpty()) { + final List existingOptions = existingField.getOptions(); if (existingOptions == null || !new HashSet<>(existingOptions).equals(new HashSet<>(desiredField.options))) { changes.put("options", desiredField.options); } @@ -380,39 +368,32 @@ private Map getFieldChanges(final MetadataTemplate.Field existin } private MetadataTemplate.FieldOperation createAddFieldOperation(final FieldDefinition field) { - final StringBuilder jsonBuilder = new StringBuilder(); - jsonBuilder.append("{\"op\":\"addField\",\"data\":{"); + // Build JSON for the addField operation + boolean isEnumOrMultiSelect = "enum".equals(field.type) || "multiSelect".equals(field.type); - // Add mandatory fields - jsonBuilder.append("\"key\":\"").append(field.key).append("\","); - jsonBuilder.append("\"type\":\"").append(field.type).append("\","); - - // Add optional fields + final StringBuilder jsonBuilder = new StringBuilder(); + jsonBuilder.append("{\"op\":\"addField\",\"data\":{") + .append("\"key\":\"").append(field.key).append("\",") + .append("\"type\":\"").append(field.type).append("\","); if (field.displayName != null) { jsonBuilder.append("\"displayName\":\"").append(field.displayName).append("\","); } - jsonBuilder.append("\"hidden\":").append(field.hidden); if (field.description != null) { jsonBuilder.append(",\"description\":\"").append(field.description).append("\""); } - // Add options for enum or multiSelect fields - if (("enum".equals(field.type) || "multiSelect".equals(field.type)) && - field.options != null && !field.options.isEmpty()) { + if (isEnumOrMultiSelect && field.options != null && !field.options.isEmpty()) { jsonBuilder.append(",\"options\":["); - for (int i = 0; i < field.options.size(); i++) { jsonBuilder.append("{\"key\":\"").append(field.options.get(i)).append("\"}"); if (i < field.options.size() - 1) { jsonBuilder.append(","); } } - jsonBuilder.append("]"); } - jsonBuilder.append("}}"); return new MetadataTemplate.FieldOperation(jsonBuilder.toString()); @@ -420,23 +401,24 @@ private MetadataTemplate.FieldOperation createAddFieldOperation(final FieldDefin private MetadataTemplate.FieldOperation createEditFieldOperation(final String fieldKey, final Map changes) { final StringBuilder jsonBuilder = new StringBuilder(); - jsonBuilder.append("{\"op\":\"editField\",\"fieldKey\":\"").append(fieldKey).append("\",\"data\":{"); + jsonBuilder.append("{\"op\":\"editField\",\"fieldKey\":\"") + .append(fieldKey).append("\",\"data\":{"); int i = 0; - for (Map.Entry entry : changes.entrySet()) { + for (final Map.Entry entry : changes.entrySet()) { if (i > 0) { jsonBuilder.append(","); } - jsonBuilder.append("\"").append(entry.getKey()).append("\":"); - if (entry.getValue() instanceof String) { - jsonBuilder.append("\"").append(entry.getValue()).append("\""); - } else if (entry.getValue() instanceof Boolean) { - jsonBuilder.append(entry.getValue()); - } else if (entry.getValue() instanceof List) { + final Object value = entry.getValue(); + if (value instanceof String) { + jsonBuilder.append("\"").append(value).append("\""); + } else if (value instanceof Boolean) { + jsonBuilder.append(value); + } else if (value instanceof List) { @SuppressWarnings("unchecked") - List options = (List) entry.getValue(); + List options = (List) value; jsonBuilder.append("["); for (int j = 0; j < options.size(); j++) { jsonBuilder.append("{\"key\":\"").append(options.get(j)).append("\"}"); @@ -446,26 +428,23 @@ private MetadataTemplate.FieldOperation createEditFieldOperation(final String fi } jsonBuilder.append("]"); } else { - jsonBuilder.append(entry.getValue()); + jsonBuilder.append(value); } - i++; } - jsonBuilder.append("}}"); return new MetadataTemplate.FieldOperation(jsonBuilder.toString()); } private MetadataTemplate.FieldOperation createRemoveFieldOperation(final String fieldKey) { - // Create the operation JSON - String removeFieldJson = String.format("{\"op\":\"removeField\",\"fieldKey\":\"%s\"}", fieldKey); - + // Build JSON for the removeField operation + final String removeFieldJson = String.format("{\"op\":\"removeField\",\"fieldKey\":\"%s\"}", fieldKey); return new MetadataTemplate.FieldOperation(removeFieldJson); } protected MetadataTemplate getMetadataTemplate(final String scope, final String templateKey) { - return MetadataTemplate.getMetadataTemplate(boxAPIConnection, scope, templateKey); + return MetadataTemplate.getMetadataTemplate(boxAPIConnection, templateKey, scope); } protected void updateMetadataTemplate(final String scope, diff --git a/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor index 90461c91e231..25e6a5d4948b 100644 --- a/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor +++ b/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor @@ -30,3 +30,4 @@ org.apache.nifi.processors.box.ListBoxFileMetadataInstances org.apache.nifi.processors.box.ListBoxFileMetadataTemplates org.apache.nifi.processors.box.PutBoxFile org.apache.nifi.processors.box.UpdateBoxFileMetadataInstance +org.apache.nifi.processors.box.UpdateBoxMetadataTemplate diff --git a/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/test/java/org/apache/nifi/processors/box/UpdateBoxMetadataTemplateTest.java b/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/test/java/org/apache/nifi/processors/box/UpdateBoxMetadataTemplateTest.java index 43b1a9540ef5..598c4b5534db 100644 --- a/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/test/java/org/apache/nifi/processors/box/UpdateBoxMetadataTemplateTest.java +++ b/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/test/java/org/apache/nifi/processors/box/UpdateBoxMetadataTemplateTest.java @@ -16,11 +16,9 @@ */ package org.apache.nifi.processors.box; -import com.box.sdk.BoxAPIConnection; import com.box.sdk.BoxAPIResponseException; import com.box.sdk.MetadataTemplate; import org.apache.nifi.json.JsonTreeReader; -import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.reporting.InitializationException; import org.apache.nifi.util.MockFlowFile; import org.apache.nifi.util.TestRunner; @@ -50,11 +48,6 @@ public class UpdateBoxMetadataTemplateTest extends AbstractBoxFileTest { private String capturedTemplateKey; private class TestUpdateBoxMetadataTemplate extends UpdateBoxMetadataTemplate { - @Override - protected BoxAPIConnection getBoxAPIConnection(final ProcessContext context) { - return mockBoxAPIConnection; - } - @Override protected MetadataTemplate getMetadataTemplate(final String scope, final String templateKey) { if (scope.equals("notFound") || templateKey.equals("notFound")) { @@ -153,9 +146,8 @@ public void testUpdateFields() { assertEquals(TEMPLATE_KEY, capturedTemplateKey); assertFalse(capturedOperations.isEmpty(), "Should generate operations for template updates"); - // Check success results testRunner.assertAllFlowFilesTransferred(UpdateBoxMetadataTemplate.REL_SUCCESS, 1); - final MockFlowFile outFile = testRunner.getFlowFilesForRelationship(UpdateBoxMetadataTemplate.REL_SUCCESS).get(0); + final MockFlowFile outFile = testRunner.getFlowFilesForRelationship(UpdateBoxMetadataTemplate.REL_SUCCESS).getFirst(); outFile.assertAttributeEquals("box.template.key", TEMPLATE_KEY); outFile.assertAttributeEquals("box.template.scope", SCOPE); outFile.assertAttributeEquals("box.template.operations.count", String.valueOf(capturedOperations.size())); @@ -180,7 +172,6 @@ public void testTemplateNotFound() { testRunner.enqueue(content); testRunner.run(); - // Verify correct routing testRunner.assertAllFlowFilesTransferred(UpdateBoxMetadataTemplate.REL_TEMPLATE_NOT_FOUND, 1); final MockFlowFile outFile = testRunner.getFlowFilesForRelationship(UpdateBoxMetadataTemplate.REL_TEMPLATE_NOT_FOUND).getFirst(); outFile.assertAttributeExists(BoxFileAttributes.ERROR_MESSAGE); @@ -201,8 +192,6 @@ public void testInvalidInputRecord() { testRunner.enqueue(content); testRunner.run(); - - // Verify correct routing testRunner.assertAllFlowFilesTransferred(UpdateBoxMetadataTemplate.REL_FAILURE, 1); } @@ -229,23 +218,18 @@ public void testNoChangesNeeded() { testRunner.enqueue(content); testRunner.run(); - - // Verify no operations generated assertEquals(0, capturedOperations.size(), "Should not generate any operations when no changes needed"); - - // Verify success flow testRunner.assertAllFlowFilesTransferred(UpdateBoxMetadataTemplate.REL_SUCCESS, 1); } @Test public void testApiError() { - // Setup test to throw error when updating testRunner.setProperty(UpdateBoxMetadataTemplate.TEMPLATE_KEY, "forbidden"); final String content = """ [ { - "key": "company_name", + "key": "company_name", "displayName": "Company Name", "type": "string", "hidden": false @@ -255,8 +239,6 @@ public void testApiError() { testRunner.enqueue(content); testRunner.run(); - - // Verify correct routing testRunner.assertAllFlowFilesTransferred(UpdateBoxMetadataTemplate.REL_FAILURE, 1); } }