diff --git a/src/main/java/in/handyman/raven/lib/CharactercountAction.java b/src/main/java/in/handyman/raven/lib/CharactercountAction.java index 790bf1fc3..73ec338b9 100644 --- a/src/main/java/in/handyman/raven/lib/CharactercountAction.java +++ b/src/main/java/in/handyman/raven/lib/CharactercountAction.java @@ -38,7 +38,7 @@ public static int getCharCount(String input, int countLimit, int threshold) { int confidenceScore = 0; try { AdapterInterface charCountAdapter = new CharacterCountAdapter(); - int wordCount = charCountAdapter.getThresoldScore(input); + int wordCount = charCountAdapter.getThresholdScore(input); confidenceScore = wordCount <= countLimit ? threshold : 0; } catch (Exception ex) { throw new HandymanException("Failed to execute char count", ex); @@ -51,7 +51,7 @@ public void execute() throws Exception { try { log.info(aMarker, "<-------Character Count Action for {} has been started------->" + charactercount.getName()); AdapterInterface charCountAdapter = new CharacterCountAdapter(); - int wordCount = charCountAdapter.getThresoldScore(charactercount.getInputValue()); + int wordCount = charCountAdapter.getThresholdScore(charactercount.getInputValue()); int confidenceScore = wordCount <= Integer.parseInt(charactercount.getCountLimit()) ? Integer.parseInt(charactercount.getThresholdValue()) : 0; action.getContext().put(charactercount.getName().concat(".score"), String.valueOf(confidenceScore)); diff --git a/src/main/java/in/handyman/raven/lib/NervalidatorAction.java b/src/main/java/in/handyman/raven/lib/NervalidatorAction.java index 314df675d..47ea86847 100644 --- a/src/main/java/in/handyman/raven/lib/NervalidatorAction.java +++ b/src/main/java/in/handyman/raven/lib/NervalidatorAction.java @@ -37,8 +37,10 @@ public NervalidatorAction(final ActionExecutionAudit action, final Logger log, } public static int getNerScore(Validator adapter, String uri) { - int confidenceScore = 0; try { + AdapterInterface nameAdapter = new NameAdapter(); + boolean validator = nameAdapter.getValidationModel(adapter.getInputValue(), uri); + return validator ? adapter.getThreshold() : 0; AdapterInterface alphaAdapter = new AlphaAdapter(); boolean alphaValidator = alphaAdapter.getValidationModel(adapter.getInputValue(), adapter.getAllowedSpecialChar()); if (alphaValidator) { @@ -49,7 +51,6 @@ public static int getNerScore(Validator adapter, String uri) { } catch (Exception ex) { throw new HandymanException("Failed to execute", ex); } - return confidenceScore; } @Override diff --git a/src/main/java/in/handyman/raven/lib/ScalarAdapterAction.java b/src/main/java/in/handyman/raven/lib/ScalarAdapterAction.java index 27dee2792..0d6ff29d3 100644 --- a/src/main/java/in/handyman/raven/lib/ScalarAdapterAction.java +++ b/src/main/java/in/handyman/raven/lib/ScalarAdapterAction.java @@ -1,7 +1,6 @@ package in.handyman.raven.lib; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; import in.handyman.raven.exception.HandymanException; import in.handyman.raven.lambda.access.ResourceAccess; @@ -22,6 +21,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -34,13 +34,13 @@ actionName = "ScalarAdapter" ) public class ScalarAdapterAction implements IActionExecution { + public static final String NER = "ner"; private final ActionExecutionAudit action; private final Logger log; private final ScalarAdapter scalarAdapter; private final Marker aMarker; - private final ObjectMapper mapper = new ObjectMapper(); public ScalarAdapterAction(final ActionExecutionAudit action, final Logger log, @@ -60,7 +60,7 @@ public void execute() throws Exception { final List validatorConfigurationDetails = new ArrayList<>(); jdbi.useTransaction(handle -> { - final List formattedQuery = CommonQueryUtil.getFormattedQuery(scalarAdapter.getResuletSet()); + final List formattedQuery = CommonQueryUtil.getFormattedQuery(scalarAdapter.getResultSet()); formattedQuery.forEach(sqlToExecute -> { validatorConfigurationDetails.addAll(handle.createQuery(sqlToExecute). mapToBean(ValidatorConfigurationDetail.class).stream().collect(Collectors.toList())); @@ -89,11 +89,16 @@ private void doProcess(final Jdbi jdbi, final List final int batchSize = validatorConfigurationDetails.size() / parallelism; if (parallelism > 1 && batchSize > 0) { log.info(aMarker, "parallel processing has started" + scalarAdapter.getName()); + final List nerValidatorConfigurationDetails = validatorConfigurationDetails.stream().filter(validatorConfigurationDetail -> Objects.equals(validatorConfigurationDetail.getAllowedAdapter(), "")).collect(Collectors.toList()); - final List> donutLineItemPartitions = Lists.partition(validatorConfigurationDetails, batchSize); - final CountDownLatch countDownLatch = new CountDownLatch(donutLineItemPartitions.size()); + validatorConfigurationDetails.removeAll(nerValidatorConfigurationDetails); + + final List> partition = Lists.partition(validatorConfigurationDetails, batchSize); + + final CountDownLatch countDownLatch = new CountDownLatch(partition.size()); final ExecutorService executorService = Executors.newFixedThreadPool(parallelism); - donutLineItemPartitions.forEach(items -> executorService.submit(() -> { + + partition.forEach(items -> executorService.submit(() -> { try { items.forEach(validatorConfigurationDetail -> { @@ -106,6 +111,11 @@ private void doProcess(final Jdbi jdbi, final List log.info(aMarker, " {} batch processed", countDownLatch.getCount()); } })); + + if (!nerValidatorConfigurationDetails.isEmpty()) { + doProcess(jdbi, nerValidatorConfigurationDetails); + } + countDownLatch.await(); } else { @@ -167,7 +177,7 @@ int computeAdapterScore(Validator inputDetail) { case "numeric": confidenceScore = NumericvalidatorAction.getNumericScore(inputDetail); break; - case "ner": + case NER: final String URI = action.getContext().get("copro.text-validation.url"); confidenceScore = NervalidatorAction.getNerScore(inputDetail, URI); break; diff --git a/src/main/java/in/handyman/raven/lib/WordcountAction.java b/src/main/java/in/handyman/raven/lib/WordcountAction.java index e8a15592c..f388ba79f 100644 --- a/src/main/java/in/handyman/raven/lib/WordcountAction.java +++ b/src/main/java/in/handyman/raven/lib/WordcountAction.java @@ -38,7 +38,7 @@ public static int getWordCount(String input, int countLimit, int threshold) { int confidenceScore = 0; try { AdapterInterface wordCountAdapter = new WordCountAdapter(); - int wordCount = wordCountAdapter.getThresoldScore(input); + int wordCount = wordCountAdapter.getThresholdScore(input); confidenceScore = wordCount <= countLimit ? threshold : 0; } catch (Exception e) { throw new HandymanException("Failed to execute", e); @@ -51,7 +51,7 @@ public void execute() throws Exception { try { log.info(aMarker, "<-------Word Count Action for {} has been started------->" + wordcount.getName()); AdapterInterface wordCountAdapter = new WordCountAdapter(); - int wordCount = wordCountAdapter.getThresoldScore(wordcount.getInputValue()); + int wordCount = wordCountAdapter.getThresholdScore(wordcount.getInputValue()); int confidenceScore = wordCount <= Integer.parseInt(wordcount.getCountLimit()) ? Integer.parseInt(wordcount.getThresholdValue()) : 0; action.getContext().put(wordcount.getName().concat(".score"), String.valueOf(confidenceScore)); diff --git a/src/main/java/in/handyman/raven/lib/adapters/AlphaAdapter.java b/src/main/java/in/handyman/raven/lib/adapters/AlphaAdapter.java index 91b8dac7e..79ede5aa4 100644 --- a/src/main/java/in/handyman/raven/lib/adapters/AlphaAdapter.java +++ b/src/main/java/in/handyman/raven/lib/adapters/AlphaAdapter.java @@ -3,25 +3,40 @@ import in.handyman.raven.lib.interfaces.AdapterInterface; import org.apache.commons.lang3.StringUtils; +import java.util.Set; +import java.util.stream.Collectors; + public class AlphaAdapter implements AdapterInterface { @Override public boolean getValidationModel(String input, String allowedCharacters) throws Exception { input = validateSpecialCharacters(allowedCharacters, input); - boolean isAlpha = StringUtils.isAlpha(input); - return isAlpha; + return StringUtils.isAlpha(input); } String validateSpecialCharacters(String specialCharacters, String input) { + return AlphaAdapter.replaceSplChars(specialCharacters, input); + } + + protected static String replaceSplChars(final String specialCharacters, String input) { + /*final char targetChar = Character.MI + final Set inputCharsSet = input.chars().mapToObj(c -> (char) c).collect(Collectors.toSet()); + final Set specialCharactersSet = specialCharacters.chars().mapToObj(c -> (char) c).collect(Collectors.toSet()); + final boolean retainAll = inputCharsSet.retainAll(specialCharactersSet); + if (retainAll) { + for (final Character chars : inputCharsSet) { + input = input.replace(chars, targetChar); + } + }*/ for (int i = 0; i < specialCharacters.length(); i++) { if (input.contains(Character.toString(specialCharacters.charAt(i)))) { input = input.replace(Character.toString(specialCharacters.charAt(i)), ""); } } - return input; + return input; } @Override - public int getThresoldScore(String sentence) throws Exception { + public int getThresholdScore(String sentence) throws Exception { return 0; } diff --git a/src/main/java/in/handyman/raven/lib/adapters/AlphaNumericAdapter.java b/src/main/java/in/handyman/raven/lib/adapters/AlphaNumericAdapter.java index 405525533..732c9439d 100644 --- a/src/main/java/in/handyman/raven/lib/adapters/AlphaNumericAdapter.java +++ b/src/main/java/in/handyman/raven/lib/adapters/AlphaNumericAdapter.java @@ -7,21 +7,15 @@ public class AlphaNumericAdapter implements AdapterInterface { @Override public boolean getValidationModel(String input, String allowedCharacters) throws Exception { input = validateSpecialCharacters(allowedCharacters, input); - boolean isAlphanumeric = StringUtils.isAlphanumeric(input); - return isAlphanumeric; + return StringUtils.isAlphanumeric(input); } String validateSpecialCharacters(String specialCharacters, String input) { - for (int i = 0; i < specialCharacters.length(); i++) { - if (input.contains(Character.toString(specialCharacters.charAt(i)))) { - input = input.replace(Character.toString(specialCharacters.charAt(i)), ""); - } - } - return input; + return AlphaAdapter.replaceSplChars(specialCharacters,input); } @Override - public int getThresoldScore(String sentence) throws Exception { + public int getThresholdScore(String sentence) throws Exception { return 0; } diff --git a/src/main/java/in/handyman/raven/lib/adapters/CharacterCountAdapter.java b/src/main/java/in/handyman/raven/lib/adapters/CharacterCountAdapter.java index 7d2adc765..1d8d5e2c4 100644 --- a/src/main/java/in/handyman/raven/lib/adapters/CharacterCountAdapter.java +++ b/src/main/java/in/handyman/raven/lib/adapters/CharacterCountAdapter.java @@ -4,7 +4,7 @@ public class CharacterCountAdapter implements AdapterInterface { @Override - public int getThresoldScore(String sentence) throws Exception { + public int getThresholdScore(String sentence) throws Exception { if (sentence == null || sentence.isEmpty()) { return 0; } diff --git a/src/main/java/in/handyman/raven/lib/adapters/DateAdapter.java b/src/main/java/in/handyman/raven/lib/adapters/DateAdapter.java index 8a3e3330f..78d571eed 100644 --- a/src/main/java/in/handyman/raven/lib/adapters/DateAdapter.java +++ b/src/main/java/in/handyman/raven/lib/adapters/DateAdapter.java @@ -50,7 +50,7 @@ public boolean getValidationModel(String sentence, String requiredFeature) throw } @Override - public int getThresoldScore(String sentence) throws Exception { + public int getThresholdScore(String sentence) throws Exception { return 0; } diff --git a/src/main/java/in/handyman/raven/lib/adapters/NameAdapter.java b/src/main/java/in/handyman/raven/lib/adapters/NameAdapter.java index d73922a25..56b83e29c 100644 --- a/src/main/java/in/handyman/raven/lib/adapters/NameAdapter.java +++ b/src/main/java/in/handyman/raven/lib/adapters/NameAdapter.java @@ -45,7 +45,7 @@ public boolean getDateValidationModel(String sentence, int comparableYear, Strin } @Override - public int getThresoldScore(String sentence) throws Exception { + public int getThresholdScore(String sentence) throws Exception { return 0; } } diff --git a/src/main/java/in/handyman/raven/lib/adapters/NumericAdapter.java b/src/main/java/in/handyman/raven/lib/adapters/NumericAdapter.java index 22bfe2851..fa6ef1d97 100644 --- a/src/main/java/in/handyman/raven/lib/adapters/NumericAdapter.java +++ b/src/main/java/in/handyman/raven/lib/adapters/NumericAdapter.java @@ -3,25 +3,22 @@ import in.handyman.raven.lib.interfaces.AdapterInterface; import org.apache.commons.lang3.StringUtils; +import java.util.Set; +import java.util.stream.Collectors; + public class NumericAdapter implements AdapterInterface { @Override public boolean getValidationModel(String input, String allowedCharacters) throws Exception { input = validateSpecialCharacters(allowedCharacters, input); - boolean isNumeric = StringUtils.isNumeric(input); - return isNumeric; + return StringUtils.isNumeric(input); } String validateSpecialCharacters(String specialCharacters, String input) { - for (int i = 0; i < specialCharacters.length(); i++) { - if (input.contains(Character.toString(specialCharacters.charAt(i)))) { - input = input.replace(Character.toString(specialCharacters.charAt(i)), ""); - } - } - return input; + return AlphaAdapter.replaceSplChars(specialCharacters,input); } @Override - public int getThresoldScore(String sentence) throws Exception { + public int getThresholdScore(String sentence) throws Exception { return 0; } diff --git a/src/main/java/in/handyman/raven/lib/adapters/WordCountAdapter.java b/src/main/java/in/handyman/raven/lib/adapters/WordCountAdapter.java index e7ef7c7fc..e3f64bee8 100644 --- a/src/main/java/in/handyman/raven/lib/adapters/WordCountAdapter.java +++ b/src/main/java/in/handyman/raven/lib/adapters/WordCountAdapter.java @@ -6,7 +6,7 @@ public class WordCountAdapter implements AdapterInterface { @Override - public int getThresoldScore(String sentence) throws Exception { + public int getThresholdScore(String sentence) throws Exception { if (sentence == null || sentence.isEmpty()) { return 0; } diff --git a/src/main/java/in/handyman/raven/lib/agadia/adapters/DateOfBirthAdapter.java b/src/main/java/in/handyman/raven/lib/agadia/adapters/DateOfBirthAdapter.java index 7e871d062..9787d5295 100644 --- a/src/main/java/in/handyman/raven/lib/agadia/adapters/DateOfBirthAdapter.java +++ b/src/main/java/in/handyman/raven/lib/agadia/adapters/DateOfBirthAdapter.java @@ -28,10 +28,10 @@ public int getConfidenceScore(String dob, AgadiaAdapter adapter) throws Exceptio confidenceScore = dobValidator ? confidenceScore + validatorThresold : confidenceScore; if (dobValidator) { - int wordCount = wordCountAdapter.getThresoldScore(dob); + int wordCount = wordCountAdapter.getThresholdScore(dob); confidenceScore = wordCount <= wcLimit ? confidenceScore + wcThresold : confidenceScore; - int charCount = charCountAdapter.getThresoldScore(dob); + int charCount = charCountAdapter.getThresholdScore(dob); confidenceScore = charCount <= charLimit ? confidenceScore + charThreshold : confidenceScore; } return confidenceScore; diff --git a/src/main/java/in/handyman/raven/lib/agadia/adapters/MemberIdAdapter.java b/src/main/java/in/handyman/raven/lib/agadia/adapters/MemberIdAdapter.java index ff3dd304f..89f49bfd0 100644 --- a/src/main/java/in/handyman/raven/lib/agadia/adapters/MemberIdAdapter.java +++ b/src/main/java/in/handyman/raven/lib/agadia/adapters/MemberIdAdapter.java @@ -23,10 +23,10 @@ public int getConfidenceScore(String memberId, AgadiaAdapter adapter) throws Exc int validatorThresold = adapter.getValidatorThreshold(); String validatorFeature = adapter.getValidatorDetail(); - int wordCount = wordCountAdapter.getThresoldScore(memberId); + int wordCount = wordCountAdapter.getThresholdScore(memberId); confidenceScore = wordCount <= wcLimit ? confidenceScore + wcThresold : confidenceScore; - int charCount = charCountAdapter.getThresoldScore(memberId); + int charCount = charCountAdapter.getThresholdScore(memberId); confidenceScore = charCount <= charLimit ? confidenceScore + charThreshold : confidenceScore; boolean idValidator = idValidatorAdapter.getValidationModel(memberId, validatorFeature); diff --git a/src/main/java/in/handyman/raven/lib/agadia/adapters/PatientNameAdapter.java b/src/main/java/in/handyman/raven/lib/agadia/adapters/PatientNameAdapter.java index 321bffe49..3cadeffad 100644 --- a/src/main/java/in/handyman/raven/lib/agadia/adapters/PatientNameAdapter.java +++ b/src/main/java/in/handyman/raven/lib/agadia/adapters/PatientNameAdapter.java @@ -29,9 +29,9 @@ public int getConfidenceScore(String patientName, AgadiaAdapter adapter) throws if (nameValidator) { String[] splitInput = patientName.split(" "); for (String name : splitInput) { - int wordCount = wordCountAdapter.getThresoldScore(name); + int wordCount = wordCountAdapter.getThresholdScore(name); formatScore = wordCount <= wcLimit ? formatScore + wcThresold : formatScore; - int charCount = charCountAdapter.getThresoldScore(name); + int charCount = charCountAdapter.getThresholdScore(name); formatScore = charCount <= charLimit ? formatScore + charThreshold : formatScore; } formatScore = formatScore / splitInput.length; diff --git a/src/main/java/in/handyman/raven/lib/interfaces/AdapterInterface.java b/src/main/java/in/handyman/raven/lib/interfaces/AdapterInterface.java index f300dfd2f..e75345ce5 100644 --- a/src/main/java/in/handyman/raven/lib/interfaces/AdapterInterface.java +++ b/src/main/java/in/handyman/raven/lib/interfaces/AdapterInterface.java @@ -1,7 +1,7 @@ package in.handyman.raven.lib.interfaces; public interface AdapterInterface { - int getThresoldScore(String sentence) throws Exception; + int getThresholdScore(String sentence) throws Exception; boolean getValidationModel(String sentence, String requiredFeature) throws Exception; diff --git a/src/main/java/in/handyman/raven/lib/model/ScalarAdapter.java b/src/main/java/in/handyman/raven/lib/model/ScalarAdapter.java index cdb0f9588..c0eefd59c 100644 --- a/src/main/java/in/handyman/raven/lib/model/ScalarAdapter.java +++ b/src/main/java/in/handyman/raven/lib/model/ScalarAdapter.java @@ -24,7 +24,7 @@ public class ScalarAdapter implements IActionContext { private String resourceConn; - private String resuletSet; + private String resultSet; private String forkBatchSize; private Boolean condition = true; diff --git a/src/test/java/in/handyman/raven/lib/ScalarAdapterActionTest.java b/src/test/java/in/handyman/raven/lib/ScalarAdapterActionTest.java new file mode 100644 index 000000000..3823d884a --- /dev/null +++ b/src/test/java/in/handyman/raven/lib/ScalarAdapterActionTest.java @@ -0,0 +1,40 @@ +package in.handyman.raven.lib; + +import in.handyman.raven.lambda.doa.audit.ActionExecutionAudit; +import in.handyman.raven.lib.model.ScalarAdapter; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Test; + +@Slf4j +class ScalarAdapterActionTest { + + @Test + void execute() throws Exception { + + final ScalarAdapter build = ScalarAdapter.builder() + .condition(true) + .name("Test ScalarAdapter") + .forkBatchSize("1") + .resultSet(" SELECT dp.sor_id, dp.sor_key , dp.question, dp.answer as input_value,\n" + + " si.allowed_adapter , si.restricted_adapter ,\n" + + " si.word_limit , si.word_threshold ,\n" + + " si.char_limit , si.char_threshold ,\n" + + " si.validator_threshold , si.allowed_characters ,\n" + + " si.comparable_characters, si.restricted_adapter_flag,\n" + + " dp.file_ref_id ,dp.paper_no ,dp.group_id, dp.sor_item_id,\n" + + " dp.created_user_id, dp.tenant_id\n" + + " FROM macro.docnet_processor dp\n" + + " JOIN sor_meta.sor_item si ON si.sor_key = dp.sor_key") + .resourceConn("intics_agadia_db_conn") + .build(); + + + final ActionExecutionAudit action = ActionExecutionAudit.builder() + .build(); + action.setRootPipelineId(11011L); + //action.getContext().put("copro.text-validation.url", "http://localhost:10189/copro/text-validation/patient"); + final ScalarAdapterAction scalarAdapterAction = new ScalarAdapterAction(action, log, build); + scalarAdapterAction.execute(); + + } +} \ No newline at end of file