diff --git a/plugins/actions/mysqlbulkload/src/main/java/org/apache/hop/workflow/actions/mysqlbulkload/ActionMysqlBulkLoad.java b/plugins/actions/mysqlbulkload/src/main/java/org/apache/hop/workflow/actions/mysqlbulkload/ActionMysqlBulkLoad.java index 54e518e92a3..6fa4c46b926 100644 --- a/plugins/actions/mysqlbulkload/src/main/java/org/apache/hop/workflow/actions/mysqlbulkload/ActionMysqlBulkLoad.java +++ b/plugins/actions/mysqlbulkload/src/main/java/org/apache/hop/workflow/actions/mysqlbulkload/ActionMysqlBulkLoad.java @@ -19,6 +19,8 @@ import java.io.File; import java.util.List; +import lombok.Getter; +import lombok.Setter; import org.apache.commons.vfs2.FileObject; import org.apache.commons.vfs2.provider.local.LocalFile; import org.apache.hop.core.Const; @@ -31,24 +33,21 @@ import org.apache.hop.core.exception.HopDatabaseException; import org.apache.hop.core.exception.HopException; import org.apache.hop.core.exception.HopFileException; -import org.apache.hop.core.exception.HopXmlException; import org.apache.hop.core.util.Utils; import org.apache.hop.core.variables.IVariables; import org.apache.hop.core.vfs.HopVfs; -import org.apache.hop.core.xml.XmlHandler; import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.metadata.api.HopMetadataProperty; import org.apache.hop.metadata.api.IHopMetadataProvider; import org.apache.hop.resource.ResourceEntry; import org.apache.hop.resource.ResourceEntry.ResourceType; import org.apache.hop.resource.ResourceReference; import org.apache.hop.workflow.WorkflowMeta; import org.apache.hop.workflow.action.ActionBase; -import org.apache.hop.workflow.action.IAction; import org.apache.hop.workflow.action.validator.AbstractFileValidator; import org.apache.hop.workflow.action.validator.ActionValidatorUtils; import org.apache.hop.workflow.action.validator.AndValidator; import org.apache.hop.workflow.action.validator.ValidatorContext; -import org.w3c.dom.Node; /** This defines a MySQL action. */ @Action( @@ -59,138 +58,76 @@ categoryDescription = "i18n:org.apache.hop.workflow:ActionCategory.Category.BulkLoading", keywords = "i18n::ActionMysqlBulkLoad.keyword", documentationUrl = "/workflow/actions/mysqlbulkload.html") -public class ActionMysqlBulkLoad extends ActionBase implements Cloneable, IAction { +@Getter +@Setter +public class ActionMysqlBulkLoad extends ActionBase { private static final Class PKG = ActionMysqlBulkLoad.class; - public static final String CONST_SPACES = " "; - public static final String CONST_TABLENAME = "tablename"; - public static final String CONST_FILENAME = "filename"; - private String schemaname; - private String tableName; - private String filename; - private String separator; - private String enclosed; - private String escaped; - private String linestarted; - private String lineterminated; - private String ignorelines; - private boolean replacedata; - private String listattribut; - private boolean localinfile; - public int prorityvalue; - private boolean addfiletoresult; - - private DatabaseMeta connection; - - public ActionMysqlBulkLoad(String n) { - super(n, ""); - tableName = null; - schemaname = null; - filename = null; - separator = null; - enclosed = null; - escaped = null; - lineterminated = null; - linestarted = null; - replacedata = true; - ignorelines = "0"; - listattribut = null; - localinfile = true; - connection = null; - addfiletoresult = false; - } - - public ActionMysqlBulkLoad() { - this(""); - } + @HopMetadataProperty(key = "schemaname") + private String schemaName; - @Override - public Object clone() { - ActionMysqlBulkLoad je = (ActionMysqlBulkLoad) super.clone(); - return je; - } + @HopMetadataProperty(key = "tablename") + private String tableName; - @Override - public String getXml() { - StringBuilder retval = new StringBuilder(200); + @HopMetadataProperty(key = "filename") + private String fileName; - retval.append(super.getXml()); - retval.append(CONST_SPACES).append(XmlHandler.addTagValue("schemaname", schemaname)); - retval.append(CONST_SPACES).append(XmlHandler.addTagValue(CONST_TABLENAME, tableName)); - retval.append(CONST_SPACES).append(XmlHandler.addTagValue(CONST_FILENAME, filename)); - retval.append(CONST_SPACES).append(XmlHandler.addTagValue("separator", separator)); - retval.append(CONST_SPACES).append(XmlHandler.addTagValue("enclosed", enclosed)); - retval.append(CONST_SPACES).append(XmlHandler.addTagValue("escaped", escaped)); - retval.append(CONST_SPACES).append(XmlHandler.addTagValue("linestarted", linestarted)); - retval.append(CONST_SPACES).append(XmlHandler.addTagValue("lineterminated", lineterminated)); + @HopMetadataProperty(key = "separator") + private String separator; - retval.append(CONST_SPACES).append(XmlHandler.addTagValue("replacedata", replacedata)); - retval.append(CONST_SPACES).append(XmlHandler.addTagValue("ignorelines", ignorelines)); - retval.append(CONST_SPACES).append(XmlHandler.addTagValue("listattribut", listattribut)); + @HopMetadataProperty(key = "enclosed") + private String enclosed; - retval.append(CONST_SPACES).append(XmlHandler.addTagValue("localinfile", localinfile)); + @HopMetadataProperty(key = "escaped") + private String escaped; - retval.append(CONST_SPACES).append(XmlHandler.addTagValue("prorityvalue", prorityvalue)); + @HopMetadataProperty(key = "linestarted") + private String lineStarted; - retval.append(CONST_SPACES).append(XmlHandler.addTagValue("addfiletoresult", addfiletoresult)); + @HopMetadataProperty(key = "lineterminated") + private String lineTerminated; - retval - .append(CONST_SPACES) - .append( - XmlHandler.addTagValue("connection", connection == null ? null : connection.getName())); + @HopMetadataProperty(key = "ignorelines") + private String ignoreLines; - return retval.toString(); - } + @HopMetadataProperty(key = "replacedata") + private boolean replaceData; - @Override - public void loadXml(Node entrynode, IHopMetadataProvider metadataProvider, IVariables variables) - throws HopXmlException { - try { - super.loadXml(entrynode); - schemaname = XmlHandler.getTagValue(entrynode, "schemaname"); - tableName = XmlHandler.getTagValue(entrynode, CONST_TABLENAME); - filename = XmlHandler.getTagValue(entrynode, CONST_FILENAME); - separator = XmlHandler.getTagValue(entrynode, "separator"); - enclosed = XmlHandler.getTagValue(entrynode, "enclosed"); - escaped = XmlHandler.getTagValue(entrynode, "escaped"); - - linestarted = XmlHandler.getTagValue(entrynode, "linestarted"); - lineterminated = XmlHandler.getTagValue(entrynode, "lineterminated"); - replacedata = "Y".equalsIgnoreCase(XmlHandler.getTagValue(entrynode, "replacedata")); - ignorelines = XmlHandler.getTagValue(entrynode, "ignorelines"); - listattribut = XmlHandler.getTagValue(entrynode, "listattribut"); - localinfile = "Y".equalsIgnoreCase(XmlHandler.getTagValue(entrynode, "localinfile")); - prorityvalue = Const.toInt(XmlHandler.getTagValue(entrynode, "prorityvalue"), -1); - String dbname = XmlHandler.getTagValue(entrynode, "connection"); - addfiletoresult = "Y".equalsIgnoreCase(XmlHandler.getTagValue(entrynode, "addfiletoresult")); - connection = DatabaseMeta.loadDatabase(metadataProvider, dbname); - } catch (HopException e) { - throw new HopXmlException("Unable to load action of type 'Mysql bulk load' from XML node", e); - } - } + @HopMetadataProperty(key = "listattribut") + private String listAttribute; - public void setTablename(String tableName) { - this.tableName = tableName; - } + @HopMetadataProperty(key = "localinfile") + private boolean localInFile; - public void setSchemaname(String schemaname) { - this.schemaname = schemaname; - } + @HopMetadataProperty(key = "prorityvalue") + public int prorityValue; - public String getSchemaname() { - return schemaname; - } + @HopMetadataProperty(key = "addfiletoresult") + private boolean addFileToResult; - public String getTablename() { - return tableName; - } + @HopMetadataProperty(key = "connection") + private String connection; - public void setDatabase(DatabaseMeta database) { - this.connection = database; + public ActionMysqlBulkLoad(String n) { + super(n, ""); + tableName = null; + schemaName = null; + fileName = null; + separator = null; + enclosed = null; + escaped = null; + lineTerminated = null; + lineStarted = null; + replaceData = true; + ignoreLines = "0"; + listAttribute = null; + localInFile = true; + connection = null; + addFileToResult = false; } - public DatabaseMeta getDatabase() { - return connection; + public ActionMysqlBulkLoad() { + this(""); } @Override @@ -216,7 +153,7 @@ public Result execute(Result previousResult, int nr) { Result result = previousResult; result.setResult(false); - String vfsFilename = resolve(filename); + String vfsFilename = resolve(fileName); // Let's check the filename ... if (!Utils.isEmpty(vfsFilename)) { @@ -245,18 +182,24 @@ public Result execute(Result previousResult, int nr) { // Here we go... back to the regular scheduled program... // File file = new File(realFilename); - if ((file.exists() && file.canRead()) || isLocalInfile() == false) { + if ((file.exists() && file.canRead()) || isLocalInFile() == false) { // User has specified an existing file, We can continue ... if (isDetailed()) { logDetailed("File [" + realFilename + "] exists."); } if (connection != null) { + DatabaseMeta databaseMeta = null; + try { + databaseMeta = DatabaseMeta.loadDatabase(getMetadataProvider(), connection); + } catch (Exception e) { + logError("Unable to load database :" + connection, e); + } // User has specified a connection, We can continue ... - try (Database db = new Database(this, this, connection)) { + try (Database db = new Database(this, this, databaseMeta)) { db.connect(); // Get schemaname - String realSchemaname = resolve(schemaname); + String realSchemaname = resolve(schemaName); // Get tablename String realTablename = resolve(tableName); @@ -267,12 +210,12 @@ public Result execute(Result previousResult, int nr) { } // Add schemaname (Most the time Schemaname.Tablename) - if (schemaname != null) { + if (schemaName != null) { realTablename = realSchemaname + "." + realTablename; } // Set the REPLACE or IGNORE - if (isReplacedata()) { + if (isReplaceData()) { replaceIgnore = "REPLACE"; } else { replaceIgnore = "IGNORE"; @@ -285,19 +228,19 @@ public Result execute(Result previousResult, int nr) { // Set list of Column if (getRealListattribut() != null) { - listOfColumn = "(" + MysqlString(getRealListattribut()) + ")"; + listOfColumn = "(" + mysqlString(getRealListattribut()) + ")"; } // Local File execution - if (isLocalInfile()) { + if (isLocalInFile()) { localExec = "LOCAL"; } // Prority - if (prorityvalue == 1) { + if (prorityValue == 1) { // LOW priorityText = "LOW_PRIORITY"; - } else if (prorityvalue == 2) { + } else if (prorityValue == 2) { // CONCURRENT priorityText = "CONCURRENT"; } @@ -434,124 +377,35 @@ public Result execute(Result previousResult, int nr) { return result; } - public boolean isReplacedata() { - return replacedata; - } - - public void setReplacedata(boolean replacedata) { - this.replacedata = replacedata; - } - - public void setLocalInfile(boolean localinfile) { - this.localinfile = localinfile; - } - - public boolean isLocalInfile() { - return localinfile; - } - - public void setFilename(String filename) { - this.filename = filename; - } - - @Override - public String getFilename() { - return filename; - } - - public void setSeparator(String separator) { - this.separator = separator; - } - - public void setLineterminated(String lineterminated) { - this.lineterminated = lineterminated; - } - - public void setLinestarted(String linestarted) { - this.linestarted = linestarted; - } - - public String getEnclosed() { - return enclosed; - } - public String getRealEnclosed() { return resolve(getEnclosed()); } - public void setEnclosed(String enclosed) { - this.enclosed = enclosed; - } - - public String getEscaped() { - return escaped; - } - public String getRealEscaped() { return resolve(getEscaped()); } - public void setEscaped(String escaped) { - this.escaped = escaped; - } - - public String getSeparator() { - return separator; - } - - public String getLineterminated() { - return lineterminated; - } - - public String getLinestarted() { - return linestarted; - } - public String getRealLinestarted() { - return resolve(getLinestarted()); + return resolve(getLineStarted()); } public String getRealLineterminated() { - return resolve(getLineterminated()); + return resolve(getLineTerminated()); } public String getRealSeparator() { return resolve(getSeparator()); } - public void setIgnorelines(String ignorelines) { - this.ignorelines = ignorelines; - } - - public String getIgnorelines() { - return ignorelines; - } - public String getRealIgnorelines() { - return resolve(getIgnorelines()); - } - - public void setListattribut(String listattribut) { - this.listattribut = listattribut; - } - - public String getListattribut() { - return listattribut; + return resolve(getIgnoreLines()); } public String getRealListattribut() { - return resolve(getListattribut()); - } - - public void setAddFileToResult(boolean addfiletoresultin) { - this.addfiletoresult = addfiletoresultin; - } - - public boolean isAddFileToResult() { - return addfiletoresult; + return resolve(getListAttribute()); } - private String MysqlString(String listcolumns) { + private String mysqlString(String listcolumns) { /* * Handle forbiden char like ' */ @@ -575,14 +429,22 @@ public List getResourceDependencies( List references = super.getResourceDependencies(variables, workflowMeta); ResourceReference reference = null; if (connection != null) { + DatabaseMeta databaseMeta = null; + try { + databaseMeta = DatabaseMeta.loadDatabase(getMetadataProvider(), connection); + } catch (Exception e) { + logError("Unable to load database :" + connection, e); + } reference = new ResourceReference(this); references.add(reference); - reference.getEntries().add(new ResourceEntry(connection.getHostname(), ResourceType.SERVER)); reference .getEntries() - .add(new ResourceEntry(connection.getDatabaseName(), ResourceType.DATABASENAME)); + .add(new ResourceEntry(databaseMeta.getHostname(), ResourceType.SERVER)); + reference + .getEntries() + .add(new ResourceEntry(databaseMeta.getDatabaseName(), ResourceType.DATABASENAME)); } - if (filename != null) { + if (fileName != null) { String realFilename = getRealFilename(); if (reference == null) { reference = new ResourceReference(this); @@ -603,12 +465,12 @@ public void check( AbstractFileValidator.putVariableSpace(ctx, getVariables()); AndValidator.putValidators( ctx, ActionValidatorUtils.notBlankValidator(), ActionValidatorUtils.fileExistsValidator()); - ActionValidatorUtils.andValidator().validate(this, CONST_FILENAME, remarks, ctx); + ActionValidatorUtils.andValidator().validate(this, "filename", remarks, ctx); ActionValidatorUtils.andValidator() .validate( this, - CONST_TABLENAME, + "tablename", remarks, AndValidator.putValidators(ActionValidatorUtils.notBlankValidator())); } diff --git a/plugins/actions/mysqlbulkload/src/main/java/org/apache/hop/workflow/actions/mysqlbulkload/ActionMysqlBulkLoadDialog.java b/plugins/actions/mysqlbulkload/src/main/java/org/apache/hop/workflow/actions/mysqlbulkload/ActionMysqlBulkLoadDialog.java index 703938617a4..6386300104c 100644 --- a/plugins/actions/mysqlbulkload/src/main/java/org/apache/hop/workflow/actions/mysqlbulkload/ActionMysqlBulkLoadDialog.java +++ b/plugins/actions/mysqlbulkload/src/main/java/org/apache/hop/workflow/actions/mysqlbulkload/ActionMysqlBulkLoadDialog.java @@ -159,7 +159,7 @@ public IAction open() { wName.setLayoutData(fdName); // Connection line - wConnection = addConnectionLine(shell, wName, action.getDatabase(), lsMod); + wConnection = addConnectionLine(shell, wName, action.getConnection(), lsMod); // Schema name line // Schema name @@ -557,14 +557,14 @@ public void widgetSelected(SelectionEvent e) { /** Copy information from the meta-data input to the dialog fields. */ public void getData() { wName.setText(Const.nullToEmpty(action.getName())); - if (action.getSchemaname() != null) { - wSchemaname.setText(action.getSchemaname()); + if (action.getSchemaName() != null) { + wSchemaname.setText(action.getSchemaName()); } - if (action.getTablename() != null) { - wTablename.setText(action.getTablename()); + if (action.getTableName() != null) { + wTablename.setText(action.getTableName()); } - if (action.getFilename() != null) { - wFilename.setText(action.getFilename()); + if (action.getFileName() != null) { + wFilename.setText(action.getFileName()); } if (action.getSeparator() != null) { wSeparator.setText(action.getSeparator()); @@ -577,36 +577,36 @@ public void getData() { if (action.getEscaped() != null) { wEscaped.setText(action.getEscaped()); } - if (action.getLinestarted() != null) { - wLinestarted.setText(action.getLinestarted()); + if (action.getLineStarted() != null) { + wLinestarted.setText(action.getLineStarted()); } - if (action.getLineterminated() != null) { - wLineterminated.setText(action.getLineterminated()); + if (action.getLineTerminated() != null) { + wLineterminated.setText(action.getLineTerminated()); } - wReplaceData.setSelection(action.isReplacedata()); + wReplaceData.setSelection(action.isReplaceData()); - wLocalInfile.setSelection(action.isLocalInfile()); + wLocalInfile.setSelection(action.isLocalInFile()); - if (action.getIgnorelines() != null) { + if (action.getIgnoreLines() != null) { - wIgnorelines.setText(action.getIgnorelines()); + wIgnorelines.setText(action.getIgnoreLines()); } else { wIgnorelines.setText("0"); } - if (action.getListattribut() != null) { - wListattribut.setText(action.getListattribut()); + if (action.getListAttribute() != null) { + wListattribut.setText(action.getListAttribute()); } - if (action.prorityvalue >= 0) { - wPriorityValue.select(action.prorityvalue); + if (action.prorityValue >= 0) { + wPriorityValue.select(action.prorityValue); } else { wPriorityValue.select(0); // NORMAL priority } - if (action.getDatabase() != null) { - wConnection.setText(action.getDatabase().getName()); + if (action.getConnection() != null) { + wConnection.setText(action.getConnection()); } wAddFileToResult.setSelection(action.isAddFileToResult()); @@ -630,20 +630,20 @@ private void ok() { return; } action.setName(wName.getText()); - action.setDatabase(getWorkflowMeta().findDatabase(wConnection.getText(), variables)); - action.setSchemaname(wSchemaname.getText()); - action.setTablename(wTablename.getText()); - action.setFilename(wFilename.getText()); + action.setConnection(wConnection.getText()); + action.setSchemaName(wSchemaname.getText()); + action.setTableName(wTablename.getText()); + action.setFileName(wFilename.getText()); action.setSeparator(wSeparator.getText()); action.setEnclosed(wEnclosed.getText()); action.setEscaped(wEscaped.getText()); - action.setLineterminated(wLineterminated.getText()); - action.setLinestarted(wLinestarted.getText()); - action.setReplacedata(wReplaceData.getSelection()); - action.setIgnorelines(wIgnorelines.getText()); - action.setListattribut(wListattribut.getText()); - action.prorityvalue = wPriorityValue.getSelectionIndex(); - action.setLocalInfile(wLocalInfile.getSelection()); + action.setLineTerminated(wLineterminated.getText()); + action.setLineStarted(wLinestarted.getText()); + action.setReplaceData(wReplaceData.getSelection()); + action.setIgnoreLines(wIgnorelines.getText()); + action.setListAttribute(wListattribut.getText()); + action.prorityValue = wPriorityValue.getSelectionIndex(); + action.setLocalInFile(wLocalInfile.getSelection()); action.setAddFileToResult(wAddFileToResult.getSelection()); diff --git a/plugins/actions/mysqlbulkload/src/test/java/org/apache/hop/workflow/actions/mysqlbulkload/WorkflowActionMysqlBulkLoadLoadSaveTest.java b/plugins/actions/mysqlbulkload/src/test/java/org/apache/hop/workflow/actions/mysqlbulkload/WorkflowActionMysqlBulkLoadLoadSaveTest.java index b412f52bffb..033ee619cb1 100644 --- a/plugins/actions/mysqlbulkload/src/test/java/org/apache/hop/workflow/actions/mysqlbulkload/WorkflowActionMysqlBulkLoadLoadSaveTest.java +++ b/plugins/actions/mysqlbulkload/src/test/java/org/apache/hop/workflow/actions/mysqlbulkload/WorkflowActionMysqlBulkLoadLoadSaveTest.java @@ -17,15 +17,17 @@ package org.apache.hop.workflow.actions.mysqlbulkload; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + import java.util.Arrays; import java.util.List; -import org.apache.hop.junit.rules.RestoreHopEngineEnvironment; +import org.apache.hop.workflow.action.ActionSerializationTestUtil; import org.apache.hop.workflow.action.loadsave.WorkflowActionLoadSaveTestSupport; -import org.junit.ClassRule; +import org.junit.jupiter.api.Test; public class WorkflowActionMysqlBulkLoadLoadSaveTest extends WorkflowActionLoadSaveTestSupport { - @ClassRule public static RestoreHopEngineEnvironment env = new RestoreHopEngineEnvironment(); @Override protected Class getActionClass() { @@ -35,20 +37,52 @@ protected Class getActionClass() { @Override protected List listAttributes() { return Arrays.asList( - "schemaname", - "tablename", - "filename", + "schemaName", + "tableName", + "fileName", "separator", "enclosed", "escaped", - "linestarted", - "lineterminated", - "replacedata", - "ignorelines", - "listattribut", - "localInfile", - "prorityvalue", + "lineStarted", + "lineTerminated", + "replaceData", + "ignoreLines", + "listAttribute", + "localInFile", + "prorityValue", "addFileToResult", - "database"); + "connection"); + } + + @Test + public void testNewSerialization() throws Exception { + ActionMysqlBulkLoad meta = + ActionSerializationTestUtil.testSerialization( + "/mysql-bulkloader-action.xml", ActionMysqlBulkLoad.class); + + assertEquals("testSchema", meta.getSchemaName()); + assertEquals("testTable", meta.getTableName()); + assertEquals("/tmp/file.csv", meta.getFileName()); + assertEquals(",", meta.getSeparator()); + assertEquals("\"", meta.getEnclosed()); + assertEquals("mysql", meta.getConnection()); + assertFalse(meta.isAddFileToResult()); + } + + @Test + public void testClone() throws Exception { + ActionMysqlBulkLoad meta = + ActionSerializationTestUtil.testSerialization( + "/mysql-bulkloader-action.xml", ActionMysqlBulkLoad.class); + + ActionMysqlBulkLoad clone = (ActionMysqlBulkLoad) meta.clone(); + + assertEquals(clone.getSchemaName(), meta.getSchemaName()); + assertEquals(clone.getTableName(), meta.getTableName()); + assertEquals(clone.getFileName(), meta.getFileName()); + assertEquals(clone.getSeparator(), meta.getSeparator()); + assertEquals(clone.getEnclosed(), meta.getEnclosed()); + assertEquals(clone.getConnection(), meta.getConnection()); + assertEquals(clone.isAddFileToResult(), meta.isAddFileToResult()); } } diff --git a/plugins/actions/mysqlbulkload/src/test/resources/mysql-bulkloader-action.xml b/plugins/actions/mysqlbulkload/src/test/resources/mysql-bulkloader-action.xml new file mode 100644 index 00000000000..dbd5a409200 --- /dev/null +++ b/plugins/actions/mysqlbulkload/src/test/resources/mysql-bulkloader-action.xml @@ -0,0 +1,43 @@ + + + + Bulk load into MySQL + + MYSQL_BULK_LOAD + + testSchema + testTable + /tmp/file.csv + , + " + + + + Y + 0 + + Y + 0 + N + mysql + N + 976 + 112 + + \ No newline at end of file