From 632a3447b17b19c00ca260aa195ded89b322c9d3 Mon Sep 17 00:00:00 2001 From: iswerya-justin Date: Tue, 20 Feb 2024 16:25:51 +0530 Subject: [PATCH 1/6] added changes in adapters for model_is enhancement --- .../handyman/raven/lib/NerAdapterAction.java | 4 ++-- .../raven/lib/ScalarAdapterAction.java | 5 +++-- .../lib/RestApiActionTestExecutionAudit.java | 22 +++++++++++++++++++ .../raven/lib/ScalarAdapterActionTest.java | 7 +++--- 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/src/main/java/in/handyman/raven/lib/NerAdapterAction.java b/src/main/java/in/handyman/raven/lib/NerAdapterAction.java index cc21f4a1b..62286d4df 100644 --- a/src/main/java/in/handyman/raven/lib/NerAdapterAction.java +++ b/src/main/java/in/handyman/raven/lib/NerAdapterAction.java @@ -34,7 +34,7 @@ actionName = "NerAdapter" ) public class NerAdapterAction implements IActionExecution { - public static final String COLUMN_LIST = "origin_id, paper_no, group_id, process_id, sor_id, sor_item_id, sor_item_name, question, answer,weight, created_user_id, tenant_id, created_on, word_score, char_score, validator_score_allowed, validator_score_negative, confidence_score, validation_name, b_box,status,stage,message,vqa_score,question_id,synonym_id,model_name,model_version"; + public static final String COLUMN_LIST = "origin_id, paper_no, group_id, process_id, sor_id, sor_item_id, sor_item_name, question, answer,weight, created_user_id, tenant_id, created_on, word_score, char_score, validator_score_allowed, validator_score_negative, confidence_score, validation_name, b_box,status,stage,message,vqa_score,question_id,synonym_id,model_name,model_version, model_registry"; private final ActionExecutionAudit action; private final Logger log; private final NerAdapter nerAdapter; @@ -61,7 +61,7 @@ public void execute() throws Exception { // build insert prepare statement with output table columns final String insertQuery = "INSERT INTO " + nerAdapter.getResultTable() + "(" + COLUMN_LIST + ")" + - " VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ,?,?, ?, ?, ?, ?, ?,?,?,? ,? ,?,?,?,?);"; + " VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ,?,?, ?, ?, ?, ?, ?,?,?,? ,? ,?,?,?,?,?);"; log.info(aMarker, "ner adapter Insert query {}", insertQuery); //3. initiate copro processor and copro urls diff --git a/src/main/java/in/handyman/raven/lib/ScalarAdapterAction.java b/src/main/java/in/handyman/raven/lib/ScalarAdapterAction.java index 2fc27aadd..ba433c1c4 100644 --- a/src/main/java/in/handyman/raven/lib/ScalarAdapterAction.java +++ b/src/main/java/in/handyman/raven/lib/ScalarAdapterAction.java @@ -267,8 +267,8 @@ void consumerBatch(final Jdbi jdbi, List resultQue resultQueue.forEach(insert -> { jdbi.useTransaction(handle -> { try { - String COLUMN_LIST = "origin_id, paper_no, group_id, process_id, sor_id, sor_item_id, sor_item_name,question, answer, weight, created_user_id, tenant_id, created_on, word_score, char_score, validator_score_allowed, validator_score_negative, confidence_score,validation_name,b_box,status,stage,message,vqa_score,question_id,synonym_id"; - String COLUMN_BINDED_LIST = ":originId, :paperNo, :groupId, :processId , :sorId, :sorItemId, :sorKey, :question ,:inputValue, :weight, :createdUserId, :tenantId, NOW(), :wordScore , :charScore , :validatorScore, :validatorNegativeScore, :confidenceScore,:allowedAdapter,:bbox,:status,:stage,:message,:vqaScore,:questionId,:synonymId"; + String COLUMN_LIST = "origin_id, paper_no, group_id, process_id, sor_id, sor_item_id, sor_item_name,question, answer, weight, created_user_id, tenant_id, created_on, word_score, char_score, validator_score_allowed, validator_score_negative, confidence_score,validation_name,b_box,status,stage,message,vqa_score,question_id,synonym_id, model_registry"; + String COLUMN_BINDED_LIST = ":originId, :paperNo, :groupId, :processId , :sorId, :sorItemId, :sorKey, :question ,:inputValue, :weight, :createdUserId, :tenantId, NOW(), :wordScore , :charScore , :validatorScore, :validatorNegativeScore, :confidenceScore,:allowedAdapter,:bbox,:status,:stage,:message,:vqaScore,:questionId,:synonymId, :modelRegistry"; Update update = handle.createUpdate(" INSERT INTO sor_transaction.adapter_result_" + scalarAdapter.getProcessID() + " ( " + COLUMN_LIST + ") " + " VALUES( " + COLUMN_BINDED_LIST + ");" + @@ -418,6 +418,7 @@ public static class ValidatorConfigurationDetail { private String message; private Integer synonymId; private Integer questionId; + private String modelRegistry; } } diff --git a/src/test/java/in/handyman/raven/lib/RestApiActionTestExecutionAudit.java b/src/test/java/in/handyman/raven/lib/RestApiActionTestExecutionAudit.java index 7eee32e71..0fc4ae02a 100644 --- a/src/test/java/in/handyman/raven/lib/RestApiActionTestExecutionAudit.java +++ b/src/test/java/in/handyman/raven/lib/RestApiActionTestExecutionAudit.java @@ -233,4 +233,26 @@ public void performanceOptUi1() log.info(request.toString()); LambdaEngine.start(request); } + + + @Test + public void modelIdTest(){ + LContext request = LContext.builder() + .pipelineName("sor.transaction.main") + .processLoadType(HRequestResolver.LoadType.FILE.name()) + .inheritedContext(Map.ofEntries( + Map.entry("tenant_id", "1"), + Map.entry("gen_group_id.group_id", "2"), + Map.entry("created_user_id", "-11"), + Map.entry("batch_id","TMP-AGD-011"), + Map.entry("document_id","TMP-AGD-111"), + Map.entry("last_updated_user_id","111"), + Map.entry("origin_type","TRANSACTION"), + Map.entry("transaction_id","TRZ-111"), + Map.entry("init_process_id.process_id", "15")) + ) + .build(); + log.info(request.toString()); + LambdaEngine.start(request); + } } diff --git a/src/test/java/in/handyman/raven/lib/ScalarAdapterActionTest.java b/src/test/java/in/handyman/raven/lib/ScalarAdapterActionTest.java index fb041f749..e1eeee1c7 100644 --- a/src/test/java/in/handyman/raven/lib/ScalarAdapterActionTest.java +++ b/src/test/java/in/handyman/raven/lib/ScalarAdapterActionTest.java @@ -22,14 +22,14 @@ void execute() throws Exception { " si.validator_threshold , si.allowed_characters ,\n" + " si.comparable_characters, si.restricted_adapter_flag,\n" + " dp.origin_id ,dp.paper_no ,dp.group_id,\n" + - " dp.created_user_id, dp.tenant_id,dp.b_box\n" + + " dp.created_user_id, dp.tenant_id,dp.b_box,dp.model_registry\n" + " FROM sor_transaction.vqa_transaction dp\n" + " JOIN sor_meta.sor_item si ON si.sor_item_name = dp.sor_item_name\n" + - " WHERE dp.group_id = '1' AND si.allowed_adapter ='ner' AND dp.answer is not null" + + " WHERE dp.group_id = '27' AND si.allowed_adapter ='ner' AND dp.answer is not null" + " AND dp.sor_item_name ='patient_name';\n" + " ") - .resourceConn("intics_agadia_db_conn") + .resourceConn("intics_zio_db_conn") .build(); @@ -39,6 +39,7 @@ void execute() throws Exception { action.setRootPipelineId(11011L); action.getContext().put("validation.multiverse-mode","true"); action.getContext().put("validation.restricted-answers","No,None of the above"); + action.getContext().put("validaiton.char-limit-count","1"); //action.getContext().put("copro.text-validation.url", "http://localhost:10189/copro/text-validation/patient"); final ScalarAdapterAction scalarAdapterAction = new ScalarAdapterAction(action, log, build); scalarAdapterAction.execute(); From 1b99374f40e656279ab829bb99e4dbdc344651c8 Mon Sep 17 00:00:00 2001 From: iswerya-justin Date: Mon, 26 Feb 2024 15:52:28 +0530 Subject: [PATCH 2/6] added changes in adapters (scalar adapter and ner adapter) for model_id enhancement --- .../handyman/raven/lib/NerAdapterAction.java | 4 +- .../raven/lib/ScalarAdapterAction.java | 49 ++++++++++--------- .../NerAdapterConsumerProcess.java | 6 +++ .../lib/model/neradaptors/NerInputTable.java | 18 ++++--- .../lib/model/neradaptors/NerOutputTable.java | 14 +++--- .../raven/lib/ScalarAdapterActionTest.java | 8 +-- .../raven/lib/TrinityModelActionTest.java | 9 ++-- .../NerAdapterActionTest.java | 48 +++++++++++------- .../TrinityModelActionTest.java | 42 +++++++++------- 9 files changed, 116 insertions(+), 82 deletions(-) diff --git a/src/main/java/in/handyman/raven/lib/NerAdapterAction.java b/src/main/java/in/handyman/raven/lib/NerAdapterAction.java index 62286d4df..2c241f495 100644 --- a/src/main/java/in/handyman/raven/lib/NerAdapterAction.java +++ b/src/main/java/in/handyman/raven/lib/NerAdapterAction.java @@ -34,7 +34,7 @@ actionName = "NerAdapter" ) public class NerAdapterAction implements IActionExecution { - public static final String COLUMN_LIST = "origin_id, paper_no, group_id, process_id, sor_id, sor_item_id, sor_item_name, question, answer,weight, created_user_id, tenant_id, created_on, word_score, char_score, validator_score_allowed, validator_score_negative, confidence_score, validation_name, b_box,status,stage,message,vqa_score,question_id,synonym_id,model_name,model_version, model_registry"; + public static final String COLUMN_LIST = "origin_id, paper_no, group_id, process_id, sor_id, sor_item_id, sor_item_name, question,question_id,synonym_id, answer,vqa_score, weight, created_user_id, tenant_id, created_on, word_score, char_score, validator_score_allowed, validator_score_negative, confidence_score, validation_name, b_box,status,stage,message,root_pipeline_id, model_name,model_version, model_registry"; private final ActionExecutionAudit action; private final Logger log; private final NerAdapter nerAdapter; @@ -61,7 +61,7 @@ public void execute() throws Exception { // build insert prepare statement with output table columns final String insertQuery = "INSERT INTO " + nerAdapter.getResultTable() + "(" + COLUMN_LIST + ")" + - " VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ,?,?, ?, ?, ?, ?, ?,?,?,? ,? ,?,?,?,?,?);"; + " VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,?,?, ?, ?, ?, ?, ?,?,?,?,?,?,?,?,?,?,?);"; log.info(aMarker, "ner adapter Insert query {}", insertQuery); //3. initiate copro processor and copro urls diff --git a/src/main/java/in/handyman/raven/lib/ScalarAdapterAction.java b/src/main/java/in/handyman/raven/lib/ScalarAdapterAction.java index ba433c1c4..db5376667 100644 --- a/src/main/java/in/handyman/raven/lib/ScalarAdapterAction.java +++ b/src/main/java/in/handyman/raven/lib/ScalarAdapterAction.java @@ -267,8 +267,8 @@ void consumerBatch(final Jdbi jdbi, List resultQue resultQueue.forEach(insert -> { jdbi.useTransaction(handle -> { try { - String COLUMN_LIST = "origin_id, paper_no, group_id, process_id, sor_id, sor_item_id, sor_item_name,question, answer, weight, created_user_id, tenant_id, created_on, word_score, char_score, validator_score_allowed, validator_score_negative, confidence_score,validation_name,b_box,status,stage,message,vqa_score,question_id,synonym_id, model_registry"; - String COLUMN_BINDED_LIST = ":originId, :paperNo, :groupId, :processId , :sorId, :sorItemId, :sorKey, :question ,:inputValue, :weight, :createdUserId, :tenantId, NOW(), :wordScore , :charScore , :validatorScore, :validatorNegativeScore, :confidenceScore,:allowedAdapter,:bbox,:status,:stage,:message,:vqaScore,:questionId,:synonymId, :modelRegistry"; + String COLUMN_LIST = "origin_id, paper_no, group_id, process_id, sor_id, sor_item_id, sor_item_name,question,question_id, synonym_id, answer,vqa_score, weight, created_user_id, tenant_id, created_on, word_score, char_score, validator_score_allowed, validator_score_negative, confidence_score,validation_name,b_box,status,stage,message,root_pipeline_id,model_name,model_version, model_registry"; + String COLUMN_BINDED_LIST = ":originId, :paperNo, :groupId, :processId , :sorId, :sorItemId, :sorKey, :question ,:questionId, :synonymId, :inputValue,:vqaScore, :weight, :createdUserId, :tenantId, NOW(), :wordScore , :charScore , :validatorScore, :validatorNegativeScore, :confidenceScore,:allowedAdapter,:bbox,:status,:stage,:message,:rootPipelineId,:modelName,:modelVersion, :modelRegistry"; Update update = handle.createUpdate(" INSERT INTO sor_transaction.adapter_result_" + scalarAdapter.getProcessID() + " ( " + COLUMN_LIST + ") " + " VALUES( " + COLUMN_BINDED_LIST + ");" + @@ -377,32 +377,26 @@ public static class SanitarySummary { } + @AllArgsConstructor @NoArgsConstructor @Data @Builder @JsonIgnoreProperties(ignoreUnknown = true) public static class ValidatorConfigurationDetail { - private int sorId; private String originId; + private int paperNo; + private Integer groupId; private String ProcessId; + private int sorId; + private int sorItemId; private String sorKey; private String question; + private Integer questionId; + private Integer synonymId; private String inputValue; - private String allowedAdapter; - private String restrictedAdapter; - private int wordLimit; - private int wordThreshold; - private int charLimit; - private int charThreshold; - private int validatorThreshold; - private String allowedCharacters; - private String comparableCharacters; - private int restrictedAdapterFlag; - private int paperNo; - private Integer groupId; - private String bbox; - private int sorItemId; + private float vqaScore; + private int weight; private String createdUserId; private Long tenantId; private double wordScore; @@ -410,15 +404,26 @@ public static class ValidatorConfigurationDetail { private double validatorScore; private double validatorNegativeScore; private double confidenceScore; - private String sorItemName; - private float vqaScore; - private int weight; + private String allowedAdapter; + private String restrictedAdapter; + private String bbox; private String status; private String stage; private String message; - private Integer synonymId; - private Integer questionId; + private Long rootPipelineId; + private String modelName; + private String modelVersion; private String modelRegistry; + + private int wordLimit; + private int wordThreshold; + private int charLimit; + private int charThreshold; + private int validatorThreshold; + private String allowedCharacters; + private String comparableCharacters; + private int restrictedAdapterFlag; + private String sorItemName; } } diff --git a/src/main/java/in/handyman/raven/lib/model/neradaptors/NerAdapterConsumerProcess.java b/src/main/java/in/handyman/raven/lib/model/neradaptors/NerAdapterConsumerProcess.java index 2a735430c..eaa6fdc09 100644 --- a/src/main/java/in/handyman/raven/lib/model/neradaptors/NerAdapterConsumerProcess.java +++ b/src/main/java/in/handyman/raven/lib/model/neradaptors/NerAdapterConsumerProcess.java @@ -114,6 +114,8 @@ public List process(URL endpoint, NerInputTable result) throws E int weight = result.getWeight(); float vqaScore = result.getVqaScore(); String createdUserId = result.getCreatedUserId(); + String modelRegistry = result.getModelRegistry(); + Long rootPipelineId = result.getRootPipelineId(); log.info(aMarker, "Build 19-validator vqa score {}", vqaScore); @@ -147,6 +149,8 @@ public List process(URL endpoint, NerInputTable result) throws E .status("COMPLETED") .stage("SCALAR_VALIDATION") .message("Ner validation macro completed") + .modelRegistry(modelRegistry) + .rootPipelineId(rootPipelineId) .build()); @@ -180,6 +184,8 @@ public List process(URL endpoint, NerInputTable result) throws E .status("FAILED") .stage("SCALAR_VALIDATION") .message("Confidence Score is less than 0") + .modelRegistry(result.getModelRegistry()) + .rootPipelineId(rootPipelineId) .build()); log.error(aMarker, "The Exception occurred in confidence score validation by {} ", valConfidenceScore); } diff --git a/src/main/java/in/handyman/raven/lib/model/neradaptors/NerInputTable.java b/src/main/java/in/handyman/raven/lib/model/neradaptors/NerInputTable.java index 649cdc1a3..b89293893 100644 --- a/src/main/java/in/handyman/raven/lib/model/neradaptors/NerInputTable.java +++ b/src/main/java/in/handyman/raven/lib/model/neradaptors/NerInputTable.java @@ -15,15 +15,15 @@ @Builder @JsonIgnoreProperties(ignoreUnknown = true) public class NerInputTable implements CoproProcessor.Entity { - private int sorId; - private String originId; - private String processId; + private int sorItemId; private String sorKey; private String question; private String inputValue; + private int weight; private float vqaScore; private String allowedAdapter; private String restrictedAdapter; + private String processId; private int wordLimit; private int wordThreshold; private int charLimit; @@ -32,21 +32,23 @@ public class NerInputTable implements CoproProcessor.Entity { private String allowedCharacters; private String comparableCharacters; private int restrictedAdapterFlag; + private String originId; private int paperNo; private Integer groupId; - private String bbox; - private int sorItemId; private String createdUserId; + private Long rootPipelineId; private Long tenantId; + private String bbox; + private Integer questionId; + private Integer synonymId; + private String modelRegistry; + private int sorId; private double wordScore; private double charScore; private double validatorScore; private double validatorNegativeScore; private double confidenceScore; private String sorItemName; - private int weight; - private Integer synonymId; - private Integer questionId; @Override public List getRowData() { diff --git a/src/main/java/in/handyman/raven/lib/model/neradaptors/NerOutputTable.java b/src/main/java/in/handyman/raven/lib/model/neradaptors/NerOutputTable.java index b28c6cc39..489ce5334 100644 --- a/src/main/java/in/handyman/raven/lib/model/neradaptors/NerOutputTable.java +++ b/src/main/java/in/handyman/raven/lib/model/neradaptors/NerOutputTable.java @@ -25,13 +25,15 @@ public class NerOutputTable implements CoproProcessor.Entity { private int sorId; private int sorItemId; private String sorItemName; - private int weight; private String question; + private Integer questionId; + private Integer synonymId; private String answer; + private float vqaScore; + private int weight; private String createdUserId; private Long tenantId; private Timestamp createdOn; - private float vqaScore; private double wordScore; private double charScore; private double validatorScoreAllowed; @@ -42,17 +44,17 @@ public class NerOutputTable implements CoproProcessor.Entity { private String status; private String stage; private String message; - private Integer synonymId; - private Integer questionId; + private Long rootPipelineId; private String modelName; private String modelVersion; + private String modelRegistry; @Override public List getRowData() { return Stream.of(this.originId, this.paperNo, this.groupId, this.processId, this.sorId, this.sorItemId, this.sorItemName, - this.question, this.answer, this.weight, this.createdUserId, this.tenantId, this.createdOn, this.wordScore, this.charScore, + this.question, this.questionId,this.synonymId,this.answer,this.vqaScore, this.weight, this.createdUserId, this.tenantId, this.createdOn, this.wordScore, this.charScore, this.validatorScoreAllowed, this.validatorScoreNegative, this.confidenceScore, this.validationName, this.bBox, - this.status, this.stage, this.message, this.vqaScore,this.questionId,this.synonymId, this.modelName, this.modelVersion + this.status, this.stage, this.message, this.rootPipelineId, this.modelName, this.modelVersion, this.modelRegistry ).collect(Collectors.toList()); } diff --git a/src/test/java/in/handyman/raven/lib/ScalarAdapterActionTest.java b/src/test/java/in/handyman/raven/lib/ScalarAdapterActionTest.java index e1eeee1c7..a83ce1c30 100644 --- a/src/test/java/in/handyman/raven/lib/ScalarAdapterActionTest.java +++ b/src/test/java/in/handyman/raven/lib/ScalarAdapterActionTest.java @@ -15,17 +15,17 @@ void execute() throws Exception { .condition(true) .name("Test ScalarAdapter") .processID("138968829607360172") - .resultSet(" SELECT dp.sor_item_name as sor_key, dp.sor_question as question, dp.answer as input_value, dp.weight,dp.vqa_score,\n" + - " si.allowed_adapter , si.restricted_adapter ,'${init_process_id.process_id}' as process_id,\n" + + .resultSet(" SELECT distinct dp.sor_item_name as sor_key,si.sor_item_id, dp.sor_question as question, dp.answer as input_value, dp.weight,dp.vqa_score,\n" + + " si.allowed_adapter , si.restricted_adapter ,dp.synonym_id, dp.question_id,'${init_process_id.process_id}' as process_id,\n" + " si.word_limit , si.word_threshold ,\n" + " si.char_limit , si.char_threshold ,\n" + " si.validator_threshold , si.allowed_characters ,\n" + " si.comparable_characters, si.restricted_adapter_flag,\n" + " dp.origin_id ,dp.paper_no ,dp.group_id,\n" + - " dp.created_user_id, dp.tenant_id,dp.b_box,dp.model_registry\n" + + " dp.created_user_id, dp.root_pipeline_id, dp.tenant_id,dp.b_box,dp.model_registry\n" + " FROM sor_transaction.vqa_transaction dp\n" + " JOIN sor_meta.sor_item si ON si.sor_item_name = dp.sor_item_name\n" + - " WHERE dp.group_id = '27' AND si.allowed_adapter ='ner' AND dp.answer is not null" + + " WHERE dp.group_id = '23' AND si.allowed_adapter ='ner' AND dp.answer is not null" + " AND dp.sor_item_name ='patient_name';\n" + " ") diff --git a/src/test/java/in/handyman/raven/lib/TrinityModelActionTest.java b/src/test/java/in/handyman/raven/lib/TrinityModelActionTest.java index 402e6e08a..9c242f53d 100644 --- a/src/test/java/in/handyman/raven/lib/TrinityModelActionTest.java +++ b/src/test/java/in/handyman/raven/lib/TrinityModelActionTest.java @@ -20,10 +20,11 @@ void executeHandwritten() throws Exception { .requestUrl("http://192.168.10.239:10189/copro/attribution/kvp-docnet") .resourceConn("intics_agadia_db_conn") .forkBatchSize("1") - .questionSql(" SELECT a.question, a.file_path, a.document_type as paperType FROM\n" + - "\t macro.sor_transaction_tqa_49254 a\n" + - "\t join sor_transaction.sor_transaction_payload_queue st on st.origin_id=a.origin_id\n" + - "\t where a.document_type='Handwritten'; ") + .questionSql(" SELECT a.question, a.file_path, a.document_type as paperType, a.model_registry FROM\n" + + "macro.sor_transaction_tqa_audit a\n" + + "join sor_transaction.sor_transaction_pipeline_audit st on st.group_id =a.group_id \n" + + "where a.document_type='Printed' and a.model_registry = 'xenon'\n" + + "and a.tenant_id = 1 and st.group_id='43'; ") .responseAs("sor_transaction_tqa_49254") .build(); ActionExecutionAudit actionExecutionAudit=new ActionExecutionAudit(); diff --git a/src/test/java/in/handyman/raven/lib/tritonservertest/NerAdapterActionTest.java b/src/test/java/in/handyman/raven/lib/tritonservertest/NerAdapterActionTest.java index 894ac8224..17212d3c3 100644 --- a/src/test/java/in/handyman/raven/lib/tritonservertest/NerAdapterActionTest.java +++ b/src/test/java/in/handyman/raven/lib/tritonservertest/NerAdapterActionTest.java @@ -17,22 +17,27 @@ void testCase() throws Exception { .name("ner") .condition(true) .resourceConn("intics_zio_db_conn") - .resultSet("SELECT 'prescriber_name' as sor_key, 'what is prescriber name' as question, 'anandh' as input_value,150 as weight,'50' as vqa_score,\n" + - " 'ner' as allowed_adapter , 'numeric_reg' as restricted_adapter ,'123456' as process_id,\n" + - " 4 as word_limit , 15 as word_threshold ,\n" + - " 60 as char_limit , 15 as char_threshold ,\n" + - " 70 as validator_threshold , '-' as allowed_characters ,\n" + - " 1 as restricted_adapter_flag,\n" + - " 'INT-1' as origin_id ,1 as paper_no ,1 as group_id,\n" + - " 1 as created_user_id, 1 as tenant_id,1 as question_id,1 as synonym_id") + .resultSet(" SELECT distinct dp.sor_item_name as sor_key,si.sor_item_id, dp.sor_question as question, dp.answer as input_value, dp.weight,dp.vqa_score,\n" + + " si.allowed_adapter , si.restricted_adapter ,dp.synonym_id, dp.question_id,'${init_process_id.process_id}' as process_id,\n" + + " si.word_limit , si.word_threshold ,\n" + + " si.char_limit , si.char_threshold ,\n" + + " si.validator_threshold , si.allowed_characters ,\n" + + " si.comparable_characters, si.restricted_adapter_flag,\n" + + " dp.origin_id ,dp.paper_no ,dp.group_id,\n" + + " dp.created_user_id, dp.root_pipeline_id, dp.tenant_id,dp.b_box,dp.model_registry\n" + + " FROM sor_transaction.vqa_transaction dp\n" + + " JOIN sor_meta.sor_item si ON si.sor_item_name = dp.sor_item_name\n" + + " WHERE dp.group_id = '23' AND si.allowed_adapter ='ner' AND dp.answer is not null" + + " AND dp.sor_item_name ='patient_name';\n" + + " ") .resultTable("sor_transaction.adapter_result_12345") .build(); ActionExecutionAudit actionExecutionAudit=new ActionExecutionAudit(); - actionExecutionAudit.getContext().put("copro.text-validation.url","http://192.168.10.239:10190/copro/text-validation/patient"); - actionExecutionAudit.setProcessId(138980079308730208L); - actionExecutionAudit.setRootPipelineId(138980079308730208L); + actionExecutionAudit.getContext().put("copro.text-validation.url","http://192.168.10.248:10190/copro/text-validation/patient"); + actionExecutionAudit.setProcessId(12345L); + actionExecutionAudit.setRootPipelineId(12345L); actionExecutionAudit.getContext().putAll(Map.ofEntries(Map.entry("read.batch.size","5"), Map.entry("ner.consumer.API.count","1"), Map.entry("validaiton.char-limit-count","10"), @@ -52,15 +57,25 @@ void tritonServer() throws Exception { .name("ner") .condition(true) .resourceConn("intics_zio_db_conn") - .resultSet("SELECT 'patient_name' as sor_key, 'what is provider name' as question, 'Christopher' as input_value, 150 as weight, allowed_adapter,restricted_adapter, '12345' as process_id" + - " ,word_limit, word_threshold, char_limit, char_threshold ,validator_threshold, allowed_characters, comparable_characters, restricted_adapter_flag,'INT-1' as origin_id ,1 as paper_no ,1 as group_id" + - " ,1 as created_user_id, 1 as tenant_id,'{\"left\":\"1234\"}' as b_box from sor_meta.sor_item where sor_item_name='patient_name'") + .resultSet(" SELECT distinct dp.sor_item_name as sor_key,si.sor_item_id, dp.sor_question as question, dp.answer as input_value, dp.weight,dp.vqa_score,\n" + + " si.allowed_adapter , si.restricted_adapter ,dp.synonym_id, dp.question_id,'12345' as process_id,\n" + + " si.word_limit , si.word_threshold ,\n" + + " si.char_limit , si.char_threshold ,\n" + + " si.validator_threshold , si.allowed_characters ,\n" + + " si.comparable_characters, si.restricted_adapter_flag,\n" + + " dp.origin_id ,dp.paper_no ,dp.group_id,\n" + + " dp.created_user_id, dp.root_pipeline_id, dp.tenant_id,dp.b_box,dp.model_registry\n" + + " FROM sor_transaction.vqa_transaction dp\n" + + " JOIN sor_meta.sor_item si ON si.sor_item_name = dp.sor_item_name\n" + + " WHERE dp.group_id = '23' AND si.allowed_adapter ='ner' AND dp.answer is not null" + + " AND dp.sor_item_name ='patient_name';\n" + + " ") .resultTable("sor_transaction.adapter_result_12345") .build(); ActionExecutionAudit actionExecutionAudit=new ActionExecutionAudit(); - actionExecutionAudit.getContext().put("copro.text-validation.url","http://192.168.10.239:10190/copro/text-validation/patient"); - actionExecutionAudit.setProcessId(138980079308730208L); + actionExecutionAudit.getContext().put("copro.text-validation.url","http://192.168.10.248:9100/v2/models/ner-service/versions/1/infer"); + actionExecutionAudit.setProcessId(12345L); actionExecutionAudit.getContext().putAll(Map.ofEntries(Map.entry("read.batch.size","5"), Map.entry("validaiton.char-limit-count","1"), Map.entry("validation.multiverse-mode","true"), @@ -69,7 +84,6 @@ void tritonServer() throws Exception { Map.entry("write.batch.size","5"), Map.entry("gen_group_id.group_id","1"), Map.entry("actionId","1"), - Map.entry("RootPipelineId","1"), Map.entry("triton.request.activator", "false"), Map.entry("paper.itemizer.consumer.API.count", "1") )); diff --git a/src/test/java/in/handyman/raven/lib/tritonservertest/TrinityModelActionTest.java b/src/test/java/in/handyman/raven/lib/tritonservertest/TrinityModelActionTest.java index 38540e055..2e21feb4d 100644 --- a/src/test/java/in/handyman/raven/lib/tritonservertest/TrinityModelActionTest.java +++ b/src/test/java/in/handyman/raven/lib/tritonservertest/TrinityModelActionTest.java @@ -18,20 +18,22 @@ void executeHandwritten() throws Exception { .name("DIE model testing") .condition(true) .outputDir("dir") - .requestUrl("http://192.168.10.239:10189/copro/attribution/kvp-docnet") - .resourceConn("intics_agadia_db_conn") + .requestUrl("http://192.168.10.248:9000/v2/models/xenon-vqa-service/versions/1/infer") + .resourceConn("intics_zio_db_conn") .forkBatchSize("1") - .questionSql(" SELECT a.question, a.file_path, a.document_type as paperType FROM\n" + - "\t macro.sor_transaction_tqa_49254 a\n" + - "\t join sor_transaction.sor_transaction_payload_queue st on st.origin_id=a.origin_id\n" + - "\t where a.document_type='Handwritten'; ") + .questionSql(" SELECT a.question, a.file_path, a.document_type as paperType FROM\n" + + "macro.sor_transaction_tqa_audit a\n" + + "join sor_transaction.sor_transaction_pipeline_audit st on st.group_id =a.group_id \n" + + "where a.document_type='Printed' and a.model_registry = 'xenon'\n" + + "and a.tenant_id = 1 and st.group_id='43'; ") .responseAs("sor_transaction_tqa_49254") .build(); ActionExecutionAudit actionExecutionAudit=new ActionExecutionAudit(); - actionExecutionAudit.getContext().put("copro.trinity-attribution.handwritten.url","http://copro.valuation:10189/copro/attribution/kvp-docnet"); + actionExecutionAudit.getContext().put("copro.trinity-attribution.handwritten.url","http://192.168.10.248:9000/v2/models/xenon-vqa-service/versions/1/infer"); actionExecutionAudit.getContext().put("okhttp.client.timeout","20"); - actionExecutionAudit.getContext().put("gen_group_id.group_id","1"); - actionExecutionAudit.setProcessId(138980079308730208L); + actionExecutionAudit.getContext().put("gen_group_id.group_id","49254"); + actionExecutionAudit.getContext().put("tenant_id", "1"); + actionExecutionAudit.setProcessId(43L); actionExecutionAudit.getContext().putAll(Map.ofEntries(Map.entry("read.batch.size","5"), Map.entry("consumer.API.count","1"), Map.entry("write.batch.size","5"))); @@ -49,20 +51,22 @@ void executePrinted() throws Exception { .name("DIE model testing") .condition(true) .outputDir("dir") - .requestUrl("http://copro.impira:10193/copro/attribution/kvp-attribution-dqa-new") - .resourceConn("intics_agadia_db_conn") + .requestUrl("http://192.168.10.248:9000/v2/models/xenon-vqa-service/versions/1/infer") + .resourceConn("intics_zio_db_conn") .forkBatchSize("1") - .questionSql(" SELECT a.question, a.file_path, a.document_type as paperType FROM\n" + - "\t macro.sor_transaction_tqa_${init_process_id.process_id} a\n" + - "\t join sor_transaction.sor_transaction_payload_queue st on st.origin_id=a.origin_id\n" + - "\t where a.document_type='Printed'; ") - .responseAs("macro.sor_transaction_tqa_49254") + .questionSql(" SELECT a.question, a.file_path, a.document_type as paperType FROM\n" + + "macro.sor_transaction_tqa_audit a\n" + + "join sor_transaction.sor_transaction_pipeline_audit st on st.group_id =a.group_id \n" + + "where a.document_type='Printed' and a.model_registry = 'xenon'\n" + + "and a.tenant_id = 1 and st.group_id='43'; ") + .responseAs("sor_transaction_tqa_49254") .build(); ActionExecutionAudit actionExecutionAudit=new ActionExecutionAudit(); - actionExecutionAudit.getContext().put("copro.trinity-attribution.printed.url","http://copro.impira:10193/copro/attribution/kvp-attribution-dqa-new"); + actionExecutionAudit.getContext().put("copro.trinity-attribution.handwritten.url","http://192.168.10.248:9000/v2/models/xenon-vqa-service/versions/1/infer"); actionExecutionAudit.getContext().put("okhttp.client.timeout","20"); - actionExecutionAudit.getContext().put("gen_group_id.group_id","1"); - actionExecutionAudit.setProcessId(138980079308730208L); + actionExecutionAudit.getContext().put("gen_group_id.group_id","49254"); + actionExecutionAudit.getContext().put("tenant_id", "1"); + actionExecutionAudit.setProcessId(43L); actionExecutionAudit.getContext().putAll(Map.ofEntries(Map.entry("read.batch.size","5"), Map.entry("consumer.API.count","1"), Map.entry("write.batch.size","5"))); From d439ec87ca6050b7d87c581d3c33071bc39ceeee Mon Sep 17 00:00:00 2001 From: iswerya-justin Date: Wed, 28 Feb 2024 18:10:54 +0530 Subject: [PATCH 3/6] added changes in trinity related files for model_id enhancement --- .../raven/lib/TrinityModelAction.java | 23 +++++++++++-------- .../trinitymodel/TrinityModelApiCaller.java | 16 +++++++++---- .../trinitymodel/TrinityModelLineItem.java | 1 + .../trinitymodel/TrinityModelPayload.java | 1 + .../trinitymodel/TrinityModelQueryResult.java | 1 + .../raven/lib/TrinityModelActionTest.java | 21 +++++++++-------- 6 files changed, 39 insertions(+), 24 deletions(-) diff --git a/src/main/java/in/handyman/raven/lib/TrinityModelAction.java b/src/main/java/in/handyman/raven/lib/TrinityModelAction.java index 7563945bc..d40baa8cf 100644 --- a/src/main/java/in/handyman/raven/lib/TrinityModelAction.java +++ b/src/main/java/in/handyman/raven/lib/TrinityModelAction.java @@ -177,6 +177,8 @@ private void computeProcess(final int nodeSize, final List private void doWork(int nodeSize, List nodes, ObjectMapper mapper, TrinityModelLineItem asset, Jdbi jdbi) { final String filePath = asset.getFilePath(); final String paperType = asset.getPaperType(); + final String modelRegistryName = asset.getModelRegistryName(); + try { @@ -185,16 +187,16 @@ private void doWork(int nodeSize, List nodes, ObjectMapper mapper, Trini if (log.isInfoEnabled()) { log.info(aMarker, "1. preparing {} for rest api call ", questions.size()); - log.info(aMarker, "2. info's are {}, {}, {}", filePath, paperType, questions); + log.info(aMarker, "2. info's are {}, {}, {}, {}", filePath, paperType, questions, modelRegistryName); } String tritonRequestActivator = action.getContext().get("triton.request.activator"); if (Objects.equals("false", tritonRequestActivator)) { - coproRequestBuilder(node,filePath,paperType,questions, jdbi, mapper); + coproRequestBuilder(node,filePath,paperType,questions,modelRegistryName, jdbi, mapper); } else { - tritonRequestBuilder(node,filePath,paperType,questions, jdbi, mapper); + tritonRequestBuilder(node,filePath,paperType,questions,modelRegistryName, jdbi, mapper); } } catch (JsonProcessingException e) { @@ -209,8 +211,8 @@ private void doWork(int nodeSize, List nodes, ObjectMapper mapper, Trini } } - private void tritonRequestBuilder(String node, String filePath, String paperType, List questions, Jdbi jdbi,ObjectMapper objectMapper) throws JsonProcessingException { - final String trinityModelResultLineItems = new TrinityModelApiCaller(this, node).computeTriton(filePath, paperType, questions, action); + private void tritonRequestBuilder(String node, String filePath, String paperType, List questions,String modelRegistryName, Jdbi jdbi,ObjectMapper objectMapper) throws JsonProcessingException { + final String trinityModelResultLineItems = new TrinityModelApiCaller(this, node).computeTriton(filePath, paperType, questions, modelRegistryName, action); TrinityModelResponse trinityModelResponse = objectMapper.readValue(trinityModelResultLineItems, new TypeReference<>() { }); trinityModelResponse.getOutputs().forEach(trinityModelOutput -> trinityModelOutput.getData().forEach(trinityModelResultLineItem -> { @@ -219,9 +221,9 @@ private void tritonRequestBuilder(String node, String filePath, String paperType })); } - private void coproRequestBuilder(String node, String filePath, String paperType, List questions, Jdbi jdbi,ObjectMapper mapper) throws JsonProcessingException { - final String trinityModelResultLineItems = new TrinityModelApiCaller(this, node).computeCopro(filePath, paperType, questions, action); - extractedCoproOutputResponse(trinityModelResultLineItems, jdbi, filePath, tenantId,paperType,"","",mapper); + private void coproRequestBuilder(String node, String filePath, String paperType, List questions,String modelRegistryName, Jdbi jdbi,ObjectMapper mapper) throws JsonProcessingException { + final String trinityModelResultLineItems = new TrinityModelApiCaller(this, node).computeCopro(filePath, paperType, questions,modelRegistryName, action); + extractedCoproOutputResponse(trinityModelResultLineItems, jdbi, filePath, tenantId,paperType,modelRegistryName,"","",mapper); } @@ -268,7 +270,7 @@ private void extractedOuputDataRequest(String trinityModelDataItems,Jdbi jdbi, S } } private void extractedCoproOutputResponse (String trinityModelDataItems, Jdbi jdbi, String filePath, Long - tenantId, String paperType, String modelName, String modelVersion, ObjectMapper objectMapper) + tenantId, String paperType, String modelRegistryName, String modelName, String modelVersion, ObjectMapper objectMapper) { try { @@ -280,7 +282,7 @@ private void extractedCoproOutputResponse (String trinityModelDataItems, Jdbi jd log.info(aMarker, "completed {}", trinityModelDataItem.getAttributes().size()); jdbi.useTransaction(handle -> { - final PreparedBatch batch = handle.prepareBatch("INSERT INTO macro." + trinityModel.getResponseAs() + " (" + COLUMN_LIST + ") VALUES(" + action.getPipelineId() + ",:filePath,:question,:predictedAttributionValue, :bBoxes::json, :imageDpi, :imageWidth, :imageHeight , :extractedImageUnit, " + action.getActionId() + "," + action.getRootPipelineId() + ",:status,:stage,:paperType, :scores, :modelName, :modelVersion,:tenantId);"); + final PreparedBatch batch = handle.prepareBatch("INSERT INTO macro." + trinityModel.getResponseAs() + " (" + COLUMN_LIST + ") VALUES(" + action.getPipelineId() + ",:filePath,:question,:predictedAttributionValue, :bBoxes::json, :imageDpi, :imageWidth, :imageHeight , :extractedImageUnit, " + action.getActionId() + "," + action.getRootPipelineId() + ",:status,:stage,:paperType, :scores, :modelName, :modelVersion,:tenantId, :modelRegistryName);"); Lists.partition(trinityModelDataItem.getAttributes(), 100).forEach(resultLineItems -> { log.info(aMarker, "inserting into trinity model_action {}", resultLineItems.size()); @@ -290,6 +292,7 @@ private void extractedCoproOutputResponse (String trinityModelDataItems, Jdbi jd .bind("predictedAttributionValue", resultLineItem.getPredictedAttributionValue()) .bind("scores", resultLineItem.getScores()) .bind("paperType", paperType) + .bind("model_registry", modelRegistryName) .bind("bBoxes", String.valueOf(resultLineItem.getBboxes())) .bind("imageDpi", trinityModelDataItem.getImageDPI()) .bind("imageWidth", trinityModelDataItem.getImageWidth()) diff --git a/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelApiCaller.java b/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelApiCaller.java index efbc83d55..5cb38d3b9 100644 --- a/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelApiCaller.java +++ b/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelApiCaller.java @@ -36,7 +36,7 @@ public TrinityModelApiCaller(TrinityModelAction aAction, final String node) { .build(); } - public String computeTriton(final String inputPath, final String paperType, final List questions, ActionExecutionAudit action) throws JsonProcessingException { + public String computeTriton(final String inputPath, final String paperType, final List questions,final String modelRegistryName, ActionExecutionAudit action) throws JsonProcessingException { Long actionId = action.getActionId(); Long rootpipelineId = action.getRootPipelineId(); @@ -51,10 +51,11 @@ public String computeTriton(final String inputPath, final String paperType, fina trinityModelPayload.setPaperType(paperType); trinityModelPayload.setAttributes(questions); trinityModelPayload.setInputFilePath(inputPath); + trinityModelPayload.setModelRegistryName(modelRegistryName); String jsonInputRequest = objectMapper.writeValueAsString(trinityModelPayload); - TritonRequest tritonRequest = getTritonRequestPaperType(paperType, jsonInputRequest); + TritonRequest tritonRequest = getTritonRequestPaperType(paperType,modelRegistryName, jsonInputRequest); TrinityModelRequest trinityModelRequest = new TrinityModelRequest(); @@ -82,14 +83,18 @@ public String computeTriton(final String inputPath, final String paperType, fina } @NotNull - private static TritonRequest getTritonRequestPaperType(String paperType, String jsonInputRequest) { + private static TritonRequest getTritonRequestPaperType(String paperType,String modelRegistryName, String jsonInputRequest) { TritonRequest tritonRequest = new TritonRequest(); if(Objects.equals(paperType,"Printed")){ tritonRequest.setShape(List.of(1, 1)); - tritonRequest.setName("ARGON VQA START"); tritonRequest.setDatatype("BYTES"); tritonRequest.setData(Collections.singletonList(jsonInputRequest)); + if (Objects.equals(modelRegistryName, "argon")) { + tritonRequest.setName("ARGON VQA START"); + } else if (Objects.equals(modelRegistryName, "xenon")){ + tritonRequest.setName("XENON VQA START"); + } } else if (Objects.equals(paperType,"Handwritten")) { tritonRequest.setShape(List.of(1, 1)); tritonRequest.setName("XENON VQA START"); @@ -100,7 +105,7 @@ private static TritonRequest getTritonRequestPaperType(String paperType, String return tritonRequest; } - public String computeCopro(final String inputPath, final String paperType, final List questions, ActionExecutionAudit action) throws JsonProcessingException { + public String computeCopro(final String inputPath, final String paperType, final List questions,final String modelRegistryName, ActionExecutionAudit action) throws JsonProcessingException { Long actionId = action.getActionId(); Long rootPipelineId = action.getRootPipelineId(); @@ -115,6 +120,7 @@ public String computeCopro(final String inputPath, final String paperType, final trinityModelPayload.setPaperType(paperType); trinityModelPayload.setAttributes(questions); trinityModelPayload.setInputFilePath(inputPath); + trinityModelPayload.setModelRegistryName(modelRegistryName); String jsonInputRequest = objectMapper.writeValueAsString(trinityModelPayload); diff --git a/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelLineItem.java b/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelLineItem.java index e9c1c4fea..3d0efeb11 100644 --- a/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelLineItem.java +++ b/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelLineItem.java @@ -18,6 +18,7 @@ public class TrinityModelLineItem { private String filePath; private List questions; private String paperType; + private String modelRegistryName; } diff --git a/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelPayload.java b/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelPayload.java index fab7e13b1..eb4c3cd99 100644 --- a/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelPayload.java +++ b/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelPayload.java @@ -23,4 +23,5 @@ public class TrinityModelPayload { private Long processId; private Integer groupId; private Long tenantId; + private String modelRegistryName; } diff --git a/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelQueryResult.java b/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelQueryResult.java index 14d6e9961..6fcfe1afd 100644 --- a/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelQueryResult.java +++ b/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelQueryResult.java @@ -16,5 +16,6 @@ public class TrinityModelQueryResult { private String filePath; private String question; private String paperType; + private String modelRegistryName; } diff --git a/src/test/java/in/handyman/raven/lib/TrinityModelActionTest.java b/src/test/java/in/handyman/raven/lib/TrinityModelActionTest.java index 9c242f53d..e87a4232e 100644 --- a/src/test/java/in/handyman/raven/lib/TrinityModelActionTest.java +++ b/src/test/java/in/handyman/raven/lib/TrinityModelActionTest.java @@ -49,20 +49,23 @@ void executePrinted() throws Exception { .name("DIE model testing") .condition(true) .outputDir("dir") - .requestUrl("http://copro.impira:10193/copro/attribution/kvp-attribution-dqa-new") - .resourceConn("intics_agadia_db_conn") + .requestUrl("http://192.168.10.248:9000/v2/models/xenon-vqa-service/versions/1/infer") + .resourceConn("intics_zio_db_conn") .forkBatchSize("1") - .questionSql(" SELECT a.question, a.file_path, a.document_type as paperType FROM\n" + - "\t macro.sor_transaction_tqa_${init_process_id.process_id} a\n" + - "\t join sor_transaction.sor_transaction_payload_queue st on st.origin_id=a.origin_id\n" + - "\t where a.document_type='Printed'; ") - .responseAs("macro.sor_transaction_tqa_49254") + .questionSql( + "SELECT distinct a.question, a.file_path, a.document_type as paperType,a.model_registry FROM\n" + + " macro.sor_transaction_tqa_audit a\n" + + " join sor_transaction.sor_transaction_pipeline_audit st on st.root_pipeline_id=a.root_pipeline_id\n" + + " where a.document_type='Printed' and a.model_registry = 'xenon'" + + "and a.root_pipeline_id = '1993'") + .responseAs("sor_transaction_tqa_49254") .build(); ActionExecutionAudit actionExecutionAudit=new ActionExecutionAudit(); - actionExecutionAudit.getContext().put("copro.trinity-attribution.printed.url","http://copro.impira:10193/copro/attribution/kvp-attribution-dqa-new"); + actionExecutionAudit.getContext().put("copro.trinity-attribution.handwritten.url","http://192.168.10.248:9000/v2/models/xenon-vqa-service/versions/1/infer"); actionExecutionAudit.getContext().put("okhttp.client.timeout","20"); actionExecutionAudit.getContext().put("gen_group_id.group_id","1"); - actionExecutionAudit.setProcessId(138980079308730208L); + actionExecutionAudit.getContext().put("tenant_id", "1"); + actionExecutionAudit.setProcessId(49254L); actionExecutionAudit.getContext().putAll(Map.ofEntries(Map.entry("read.batch.size","5"), Map.entry("consumer.API.count","1"), Map.entry("write.batch.size","5"))); From 464aadb3d7a0a1a33afa64394be97acd4ef26a9a Mon Sep 17 00:00:00 2001 From: iswerya-justin Date: Wed, 28 Feb 2024 22:50:41 +0530 Subject: [PATCH 4/6] added changes in scalar adapter --- src/main/java/in/handyman/raven/lib/ScalarAdapterAction.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/in/handyman/raven/lib/ScalarAdapterAction.java b/src/main/java/in/handyman/raven/lib/ScalarAdapterAction.java index 737805a8f..5575f4a57 100644 --- a/src/main/java/in/handyman/raven/lib/ScalarAdapterAction.java +++ b/src/main/java/in/handyman/raven/lib/ScalarAdapterAction.java @@ -270,8 +270,6 @@ void consumerBatch(final Jdbi jdbi, List resultQue try { String COLUMN_LIST = "origin_id, paper_no, group_id, process_id, sor_id, sor_item_id, sor_item_name,question,question_id, synonym_id, answer,vqa_score, weight, created_user_id, tenant_id, created_on, word_score, char_score, validator_score_allowed, validator_score_negative, confidence_score,validation_name,b_box,status,stage,message,root_pipeline_id,model_name,model_version, model_registry"; String COLUMN_BINDED_LIST = ":originId, :paperNo, :groupId, :processId , :sorId, :sorItemId, :sorKey, :question ,:questionId, :synonymId, :inputValue,:vqaScore, :weight, :createdUserId, :tenantId, NOW(), :wordScore , :charScore , :validatorScore, :validatorNegativeScore, :confidenceScore,:allowedAdapter,:bbox,:status,:stage,:message,:rootPipelineId,:modelName,:modelVersion, :modelRegistry"; - String COLUMN_LIST = "root_pipeline_id, origin_id, paper_no, group_id, process_id, sor_id, sor_item_id, sor_item_name,question, answer, weight, created_user_id, tenant_id, created_on, word_score, char_score, validator_score_allowed, validator_score_negative, confidence_score,validation_name,b_box,status,stage,message,vqa_score,question_id,synonym_id"; - String COLUMN_BINDED_LIST = ":rootPipeLineId, :originId, :paperNo, :groupId, :processId , :sorId, :sorItemId, :sorKey, :question ,:inputValue, :weight, :createdUserId, :tenantId, NOW(), :wordScore , :charScore , :validatorScore, :validatorNegativeScore, :confidenceScore,:allowedAdapter,:bbox,:status,:stage,:message,:vqaScore,:questionId,:synonymId"; Update update = handle.createUpdate(" INSERT INTO sor_transaction.adapter_result_" + scalarAdapter.getProcessID() + " ( " + COLUMN_LIST + ") " + " VALUES( " + COLUMN_BINDED_LIST + ");" + From 9e29936f10147bf50618ac612a250ce66b143c8f Mon Sep 17 00:00:00 2001 From: iswerya-justin Date: Fri, 1 Mar 2024 21:51:02 +0530 Subject: [PATCH 5/6] added changes in trinity related files for model_id enhancement --- .../raven/lib/TrinityModelAction.java | 77 ++++++++++++------- .../trinitymodel/TrinityModelApiCaller.java | 20 ++--- .../trinitymodel/TrinityModelDataItem.java | 1 + .../trinitymodel/TrinityModelLineItem.java | 3 +- .../trinitymodel/TrinityModelPayload.java | 2 +- .../trinitymodel/TrinityModelQueryResult.java | 7 +- .../TrinityModelResultLineItem.java | 1 + .../raven/lib/TrinityModelActionTest.java | 10 ++- 8 files changed, 76 insertions(+), 45 deletions(-) diff --git a/src/main/java/in/handyman/raven/lib/TrinityModelAction.java b/src/main/java/in/handyman/raven/lib/TrinityModelAction.java index d40baa8cf..82b71a5b1 100644 --- a/src/main/java/in/handyman/raven/lib/TrinityModelAction.java +++ b/src/main/java/in/handyman/raven/lib/TrinityModelAction.java @@ -34,8 +34,8 @@ actionName = "TrinityModel" ) public class TrinityModelAction implements IActionExecution { - public static final String COLUMN_LIST = "process_id,file_path,question, predicted_attribution_value,b_box, image_dpi , image_width , image_height , extracted_image_unit , action_id, root_pipeline_id,status,stage,paper_type, score,model_name,model_version,tenant_id"; - public static final String CREATE_TABLE_COLUMN = "id bigserial not null, file_path text,question text, predicted_attribution_value text , score float8 NULL, b_box json null, image_dpi int8 null, image_width int8 null, image_height int8 null, extracted_image_unit varchar null, action_id bigint, root_pipeline_id bigint,process_id bigint, created_on timestamp not null default now(),status varchar NULL,stage varchar NULL ,paper_type varchar NULL,tenant_id int8 null"; + public static final String COLUMN_LIST = "process_id,file_path,question, predicted_attribution_value,b_box, image_dpi , image_width , image_height , extracted_image_unit , action_id, root_pipeline_id,status,stage,paper_type, score,model_name,model_version,tenant_id, model_registry"; + public static final String CREATE_TABLE_COLUMN = "id bigserial not null, file_path text,question text, predicted_attribution_value text , score float8 NULL, b_box json null, image_dpi int8 null, image_width int8 null, image_height int8 null, extracted_image_unit varchar null, action_id bigint, root_pipeline_id bigint,process_id bigint, created_on timestamp not null default now(),status varchar NULL,stage varchar NULL ,paper_type varchar NULL,tenant_id int8 null, model_registry varchar NULL,model_name varchar null,model_version varchar null"; public static final String CREATE_ERROR_TABLE_COLUMN = "id bigserial not null, file_path text,error_message text, action_id bigint, root_pipeline_id bigint,process_id bigint, created_on timestamp not null default now() ,tenant_id int8 null"; private final ActionExecutionAudit action; @@ -69,6 +69,7 @@ public void execute() throws Exception { final Jdbi jdbi = ResourceAccess.rdbmsJDBIConn(trinityModel.getResourceConn()); final List trinityModelQueryResults = new ArrayList<>(); + jdbi.useTransaction(handle -> { final List formattedQuery = CommonQueryUtil.getFormattedQuery(trinityModel.getQuestionSql()); formattedQuery.forEach(sqlToExecute -> trinityModelQueryResults.addAll(handle.createQuery(sqlToExecute) @@ -82,14 +83,24 @@ public void execute() throws Exception { jdbi.useTransaction(handle -> handle.execute("create table if not exists macro." + trinityModel.getResponseAs() + " ( " + CREATE_TABLE_COLUMN + ");")); jdbi.useTransaction(handle -> handle.execute("create table if not exists macro." + trinityModel.getResponseAs() + "_error ( " + CREATE_ERROR_TABLE_COLUMN + ");")); final List trinityModelLineItems = new ArrayList<>(); + trinityModelQueryResults.stream().forEach(trinityModelQueryResult -> { + trinityModelLineItems.add(TrinityModelLineItem.builder() + .questions(trinityModelQueryResult.getQuestions()) + .filePath(trinityModelQueryResult.getFilePath()) + .paperType(trinityModelQueryResult.getPaperType()) + .modelRegistry(trinityModelQueryResult.getModelRegistry()) + .tenantId(trinityModelQueryResult.getTenantId()) + .build()); + }); - Map>> listTrinityModelQueryResult = trinityModelQueryResults.stream().collect(Collectors.groupingBy(TrinityModelQueryResult::getFilePath, Collectors.groupingBy(TrinityModelQueryResult::getPaperType))); +// Map>> listTrinityModelQueryResult = trinityModelQueryResults.stream().collect(Collectors.groupingBy(TrinityModelQueryResult::getFilePath, Collectors.groupingBy(TrinityModelQueryResult::getPaperType))); ObjectMapper mapper = new ObjectMapper(); - listTrinityModelQueryResult.forEach((s, stringListMap) -> { - stringListMap.forEach((s1, trinityModelQueryResults1) -> trinityModelLineItems.add(TrinityModelLineItem.builder() - .filePath(s).paperType(s1).questions(trinityModelQueryResults1.stream().map(TrinityModelQueryResult::getQuestion).collect(Collectors.toList())) - .build())); - }); +// listTrinityModelQueryResult.forEach((s, stringListMap) -> { +// stringListMap.forEach((s1, trinityModelQueryResults1) -> trinityModelLineItems.add(TrinityModelLineItem.builder() +// .filePath(s).paperType(s1).questions(trinityModelQueryResults1.stream().map(TrinityModelQueryResult::getQuestion).collect(Collectors.toList())) +// .build())); +// }); + String trinityUrl = trinityModel.getRequestUrl(); final List nodes = Optional.ofNullable(trinityUrl).map(s -> Arrays.asList(s.split(","))).orElse(Collections.emptyList()); @@ -177,7 +188,7 @@ private void computeProcess(final int nodeSize, final List private void doWork(int nodeSize, List nodes, ObjectMapper mapper, TrinityModelLineItem asset, Jdbi jdbi) { final String filePath = asset.getFilePath(); final String paperType = asset.getPaperType(); - final String modelRegistryName = asset.getModelRegistryName(); + final String modelRegistry = asset.getModelRegistry(); @@ -187,16 +198,16 @@ private void doWork(int nodeSize, List nodes, ObjectMapper mapper, Trini if (log.isInfoEnabled()) { log.info(aMarker, "1. preparing {} for rest api call ", questions.size()); - log.info(aMarker, "2. info's are {}, {}, {}, {}", filePath, paperType, questions, modelRegistryName); + log.info(aMarker, "2. info's are {}, {}, {}, {}", filePath, paperType, questions, modelRegistry); } String tritonRequestActivator = action.getContext().get("triton.request.activator"); if (Objects.equals("false", tritonRequestActivator)) { - coproRequestBuilder(node,filePath,paperType,questions,modelRegistryName, jdbi, mapper); + coproRequestBuilder(node,filePath,paperType,questions,modelRegistry, jdbi, mapper); } else { - tritonRequestBuilder(node,filePath,paperType,questions,modelRegistryName, jdbi, mapper); + tritonRequestBuilder(node,filePath,paperType,questions,modelRegistry, jdbi, mapper); } } catch (JsonProcessingException e) { @@ -211,23 +222,22 @@ private void doWork(int nodeSize, List nodes, ObjectMapper mapper, Trini } } - private void tritonRequestBuilder(String node, String filePath, String paperType, List questions,String modelRegistryName, Jdbi jdbi,ObjectMapper objectMapper) throws JsonProcessingException { - final String trinityModelResultLineItems = new TrinityModelApiCaller(this, node).computeTriton(filePath, paperType, questions, modelRegistryName, action); + private void tritonRequestBuilder(String node, String filePath, String paperType, List questions,String modelRegistry, Jdbi jdbi,ObjectMapper objectMapper) throws JsonProcessingException { + final String trinityModelResultLineItems = new TrinityModelApiCaller(this, node).computeTriton(filePath, paperType, questions, modelRegistry, tenantId, action); TrinityModelResponse trinityModelResponse = objectMapper.readValue(trinityModelResultLineItems, new TypeReference<>() { }); trinityModelResponse.getOutputs().forEach(trinityModelOutput -> trinityModelOutput.getData().forEach(trinityModelResultLineItem -> { - extractedOuputDataRequest(trinityModelResultLineItem, jdbi, filePath,tenantId, paperType, trinityModelResponse.getModelName(),trinityModelResponse.getModelVersion(),objectMapper); - + extractedTritonOuputDataResponse(trinityModelResultLineItem, jdbi, filePath,tenantId, paperType, trinityModelResponse.getModelName(),trinityModelResponse.getModelVersion(),modelRegistry,objectMapper); })); } - private void coproRequestBuilder(String node, String filePath, String paperType, List questions,String modelRegistryName, Jdbi jdbi,ObjectMapper mapper) throws JsonProcessingException { - final String trinityModelResultLineItems = new TrinityModelApiCaller(this, node).computeCopro(filePath, paperType, questions,modelRegistryName, action); - extractedCoproOutputResponse(trinityModelResultLineItems, jdbi, filePath, tenantId,paperType,modelRegistryName,"","",mapper); + private void coproRequestBuilder(String node, String filePath, String paperType, List questions,String modelRegistry, Jdbi jdbi,ObjectMapper mapper) throws JsonProcessingException { + final String trinityModelResultLineItems = new TrinityModelApiCaller(this, node).computeCopro(filePath, paperType, questions,modelRegistry,tenantId, action); + extractedCoproOutputResponse(trinityModelResultLineItems, jdbi, filePath, tenantId,paperType,modelRegistry,"","",mapper); } - private void extractedOuputDataRequest(String trinityModelDataItems,Jdbi jdbi, String filePath,Long tenantId, String paperType,String modelName,String modelVersion,ObjectMapper objectMapper) { + private void extractedTritonOuputDataResponse(String trinityModelDataItems, Jdbi jdbi, String filePath, Long tenantId, String paperType, String modelName, String modelVersion, String modelRegistry, ObjectMapper objectMapper) { try { @@ -238,7 +248,7 @@ private void extractedOuputDataRequest(String trinityModelDataItems,Jdbi jdbi, S log.info(aMarker, "completed {}", trinityModelDataItem.getAttributes().size()); jdbi.useTransaction(handle -> { - final PreparedBatch batch = handle.prepareBatch("INSERT INTO macro." + trinityModel.getResponseAs() + " (" + COLUMN_LIST + ") VALUES(" + action.getPipelineId() + ",:filePath,:question,:predictedAttributionValue, :bBoxes::json, :imageDpi, :imageWidth, :imageHeight , :extractedImageUnit, " + action.getActionId() + "," + action.getRootPipelineId() + ",:status,:stage,:paperType, :scores, :modelName, :modelVersion,:tenantId);"); + final PreparedBatch batch = handle.prepareBatch("INSERT INTO macro." + trinityModel.getResponseAs() + " (" + COLUMN_LIST + ") VALUES(" + action.getPipelineId() + ",:filePath,:question,:predictedAttributionValue, :bBoxes::json, :imageDpi, :imageWidth, :imageHeight , :extractedImageUnit, " + action.getActionId() + "," + action.getRootPipelineId() + ",:status,:stage,:paperType, :scores, :modelName, :modelVersion,:tenantId, :modelRegistry);"); Lists.partition(trinityModelDataItem.getAttributes(), 100).forEach(resultLineItems -> { log.info(aMarker, "inserting into trinity model_action {}", resultLineItems.size()); @@ -258,11 +268,16 @@ private void extractedOuputDataRequest(String trinityModelDataItems,Jdbi jdbi, S .bind("modelName", modelName) .bind("modelVersion", modelVersion) .bind("tenantId", resultLineItem.getTenantId()) + .bind("modelRegistry", modelRegistry) .add(); }); - int[] counts = batch.execute(); - log.info(aMarker, " persisted {} in trinity model_action", counts); + try{ + int[] counts = batch.execute(); + log.info(aMarker, " persisted {} in trinity model_action", counts); + }catch (Exception e){ + log.info(aMarker, " persisted {} in trinity model_action", e); + } }); }); } catch (JsonProcessingException e) { @@ -270,7 +285,7 @@ private void extractedOuputDataRequest(String trinityModelDataItems,Jdbi jdbi, S } } private void extractedCoproOutputResponse (String trinityModelDataItems, Jdbi jdbi, String filePath, Long - tenantId, String paperType, String modelRegistryName, String modelName, String modelVersion, ObjectMapper objectMapper) + tenantId, String paperType, String modelRegistry, String modelName, String modelVersion, ObjectMapper objectMapper) { try { @@ -282,7 +297,7 @@ private void extractedCoproOutputResponse (String trinityModelDataItems, Jdbi jd log.info(aMarker, "completed {}", trinityModelDataItem.getAttributes().size()); jdbi.useTransaction(handle -> { - final PreparedBatch batch = handle.prepareBatch("INSERT INTO macro." + trinityModel.getResponseAs() + " (" + COLUMN_LIST + ") VALUES(" + action.getPipelineId() + ",:filePath,:question,:predictedAttributionValue, :bBoxes::json, :imageDpi, :imageWidth, :imageHeight , :extractedImageUnit, " + action.getActionId() + "," + action.getRootPipelineId() + ",:status,:stage,:paperType, :scores, :modelName, :modelVersion,:tenantId, :modelRegistryName);"); + final PreparedBatch batch = handle.prepareBatch("INSERT INTO macro." + trinityModel.getResponseAs() + " (" + COLUMN_LIST + ") VALUES(" + action.getPipelineId() + ",:filePath,:question,:predictedAttributionValue, :bBoxes::json, :imageDpi, :imageWidth, :imageHeight , :extractedImageUnit, " + action.getActionId() + "," + action.getRootPipelineId() + "," + action.getProcessId() +",:status,:stage,:paperType, :scores, :modelName, :modelVersion,:tenantId, :modelRegistry);"); Lists.partition(trinityModelDataItem.getAttributes(), 100).forEach(resultLineItems -> { log.info(aMarker, "inserting into trinity model_action {}", resultLineItems.size()); @@ -292,7 +307,6 @@ private void extractedCoproOutputResponse (String trinityModelDataItems, Jdbi jd .bind("predictedAttributionValue", resultLineItem.getPredictedAttributionValue()) .bind("scores", resultLineItem.getScores()) .bind("paperType", paperType) - .bind("model_registry", modelRegistryName) .bind("bBoxes", String.valueOf(resultLineItem.getBboxes())) .bind("imageDpi", trinityModelDataItem.getImageDPI()) .bind("imageWidth", trinityModelDataItem.getImageWidth()) @@ -303,11 +317,18 @@ private void extractedCoproOutputResponse (String trinityModelDataItems, Jdbi jd .bind("modelName", modelName) .bind("modelVersion", modelVersion) .bind("tenantId", tenantId) + .bind("modelRegistry", modelRegistry) .add(); }); - int[] counts = batch.execute(); - log.info(aMarker, " persisted {} in trinity model_action", counts); + try{ + int[] counts = batch.execute(); + log.info(aMarker, " persisted {} in trinity model_action", counts); + }catch (Exception e){ + log.info(aMarker, " persisted {} in trinity model_action", e); + } + + }); }); } catch (JsonProcessingException e) { diff --git a/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelApiCaller.java b/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelApiCaller.java index 5cb38d3b9..4b0ffd94a 100644 --- a/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelApiCaller.java +++ b/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelApiCaller.java @@ -36,7 +36,7 @@ public TrinityModelApiCaller(TrinityModelAction aAction, final String node) { .build(); } - public String computeTriton(final String inputPath, final String paperType, final List questions,final String modelRegistryName, ActionExecutionAudit action) throws JsonProcessingException { + public String computeTriton(final String inputPath, final String paperType, final List questions,final String modelRegistry, final Long tenantId, ActionExecutionAudit action) throws JsonProcessingException { Long actionId = action.getActionId(); Long rootpipelineId = action.getRootPipelineId(); @@ -49,13 +49,14 @@ public String computeTriton(final String inputPath, final String paperType, fina trinityModelPayload.setProcess(trinityProcessName); trinityModelPayload.setRootPipelineId(rootpipelineId); trinityModelPayload.setPaperType(paperType); + trinityModelPayload.setTenantId(tenantId); trinityModelPayload.setAttributes(questions); trinityModelPayload.setInputFilePath(inputPath); - trinityModelPayload.setModelRegistryName(modelRegistryName); + trinityModelPayload.setModelRegistry(modelRegistry); String jsonInputRequest = objectMapper.writeValueAsString(trinityModelPayload); - TritonRequest tritonRequest = getTritonRequestPaperType(paperType,modelRegistryName, jsonInputRequest); + TritonRequest tritonRequest = getTritonRequestPaperType(paperType,modelRegistry, jsonInputRequest); TrinityModelRequest trinityModelRequest = new TrinityModelRequest(); @@ -83,19 +84,19 @@ public String computeTriton(final String inputPath, final String paperType, fina } @NotNull - private static TritonRequest getTritonRequestPaperType(String paperType,String modelRegistryName, String jsonInputRequest) { + private static TritonRequest getTritonRequestPaperType(String paperType,String modelRegistry, String jsonInputRequest) { TritonRequest tritonRequest = new TritonRequest(); if(Objects.equals(paperType,"Printed")){ tritonRequest.setShape(List.of(1, 1)); tritonRequest.setDatatype("BYTES"); tritonRequest.setData(Collections.singletonList(jsonInputRequest)); - if (Objects.equals(modelRegistryName, "argon")) { + if (Objects.equals(modelRegistry, "argon")) { tritonRequest.setName("ARGON VQA START"); - } else if (Objects.equals(modelRegistryName, "xenon")){ + } else if (Objects.equals(modelRegistry, "xenon")){ tritonRequest.setName("XENON VQA START"); } - } else if (Objects.equals(paperType,"Handwritten")) { + }else if (Objects.equals(paperType,"Handwritten")) { tritonRequest.setShape(List.of(1, 1)); tritonRequest.setName("XENON VQA START"); tritonRequest.setDatatype("BYTES"); @@ -105,7 +106,7 @@ private static TritonRequest getTritonRequestPaperType(String paperType,String m return tritonRequest; } - public String computeCopro(final String inputPath, final String paperType, final List questions,final String modelRegistryName, ActionExecutionAudit action) throws JsonProcessingException { + public String computeCopro(final String inputPath, final String paperType, final List questions,final String modelRegistry,final Long tenantId, ActionExecutionAudit action) throws JsonProcessingException { Long actionId = action.getActionId(); Long rootPipelineId = action.getRootPipelineId(); @@ -120,7 +121,8 @@ public String computeCopro(final String inputPath, final String paperType, final trinityModelPayload.setPaperType(paperType); trinityModelPayload.setAttributes(questions); trinityModelPayload.setInputFilePath(inputPath); - trinityModelPayload.setModelRegistryName(modelRegistryName); + trinityModelPayload.setModelRegistry(modelRegistry); + trinityModelPayload.setTenantId(tenantId); String jsonInputRequest = objectMapper.writeValueAsString(trinityModelPayload); diff --git a/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelDataItem.java b/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelDataItem.java index 647686b65..06ab7d8c4 100644 --- a/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelDataItem.java +++ b/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelDataItem.java @@ -20,5 +20,6 @@ public class TrinityModelDataItem { private Integer imageHeight; private String extractedImageUnit; private String inputFilePath; + private Long tenantId; } diff --git a/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelLineItem.java b/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelLineItem.java index 3d0efeb11..6ddb67bde 100644 --- a/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelLineItem.java +++ b/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelLineItem.java @@ -18,7 +18,8 @@ public class TrinityModelLineItem { private String filePath; private List questions; private String paperType; - private String modelRegistryName; + private String modelRegistry; + private Long tenantId; } diff --git a/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelPayload.java b/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelPayload.java index eb4c3cd99..728abafd9 100644 --- a/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelPayload.java +++ b/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelPayload.java @@ -23,5 +23,5 @@ public class TrinityModelPayload { private Long processId; private Integer groupId; private Long tenantId; - private String modelRegistryName; + private String modelRegistry; } diff --git a/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelQueryResult.java b/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelQueryResult.java index 6fcfe1afd..b45f7c099 100644 --- a/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelQueryResult.java +++ b/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelQueryResult.java @@ -6,6 +6,8 @@ import lombok.Data; import lombok.NoArgsConstructor; +import java.util.List; + @AllArgsConstructor @NoArgsConstructor @Data @@ -14,8 +16,9 @@ public class TrinityModelQueryResult { private String filePath; - private String question; + private List questions; private String paperType; - private String modelRegistryName; + private String modelRegistry; + private Long tenantId; } diff --git a/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelResultLineItem.java b/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelResultLineItem.java index 19fd7359c..f8a7d9576 100644 --- a/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelResultLineItem.java +++ b/src/main/java/in/handyman/raven/lib/model/trinitymodel/TrinityModelResultLineItem.java @@ -25,6 +25,7 @@ public class TrinityModelResultLineItem { private Long processId; private Integer groupId; private Long tenantId; + private String modelRegistry; } diff --git a/src/test/java/in/handyman/raven/lib/TrinityModelActionTest.java b/src/test/java/in/handyman/raven/lib/TrinityModelActionTest.java index e87a4232e..2075c3199 100644 --- a/src/test/java/in/handyman/raven/lib/TrinityModelActionTest.java +++ b/src/test/java/in/handyman/raven/lib/TrinityModelActionTest.java @@ -53,18 +53,20 @@ void executePrinted() throws Exception { .resourceConn("intics_zio_db_conn") .forkBatchSize("1") .questionSql( - "SELECT distinct a.question, a.file_path, a.document_type as paperType,a.model_registry FROM\n" + + "SELECT array_agg(a.question) as questions, a.file_path, a.document_type as paper_type, a.model_registry as model_registry, a.tenant_id FROM\n" + " macro.sor_transaction_tqa_audit a\n" + - " join sor_transaction.sor_transaction_pipeline_audit st on st.root_pipeline_id=a.root_pipeline_id\n" + " where a.document_type='Printed' and a.model_registry = 'xenon'" + - "and a.root_pipeline_id = '1993'") - .responseAs("sor_transaction_tqa_49254") + "and a.root_pipeline_id = '1993' group by a.file_path, paper_type, a.model_registry, a.tenant_id") + .responseAs("docnet_attribution_response_123") .build(); ActionExecutionAudit actionExecutionAudit=new ActionExecutionAudit(); actionExecutionAudit.getContext().put("copro.trinity-attribution.handwritten.url","http://192.168.10.248:9000/v2/models/xenon-vqa-service/versions/1/infer"); actionExecutionAudit.getContext().put("okhttp.client.timeout","20"); actionExecutionAudit.getContext().put("gen_group_id.group_id","1"); actionExecutionAudit.getContext().put("tenant_id", "1"); + actionExecutionAudit.setActionId(12L); + actionExecutionAudit.setRootPipelineId(1993L); + actionExecutionAudit.setPipelineId(12345L); actionExecutionAudit.setProcessId(49254L); actionExecutionAudit.getContext().putAll(Map.ofEntries(Map.entry("read.batch.size","5"), Map.entry("consumer.API.count","1"), From fc3ecb138752ca44de5a6d5de0611fdf77c95e97 Mon Sep 17 00:00:00 2001 From: iswerya-justin Date: Mon, 4 Mar 2024 13:21:35 +0530 Subject: [PATCH 6/6] fixed error in scalar adapter. rootPipelineId was repeated twice in POJO --- src/main/java/in/handyman/raven/lib/ScalarAdapterAction.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/in/handyman/raven/lib/ScalarAdapterAction.java b/src/main/java/in/handyman/raven/lib/ScalarAdapterAction.java index 5575f4a57..473ea56ef 100644 --- a/src/main/java/in/handyman/raven/lib/ScalarAdapterAction.java +++ b/src/main/java/in/handyman/raven/lib/ScalarAdapterAction.java @@ -205,7 +205,7 @@ private void doCompute(final Jdbi jdbi, List listO result.setCharScore(charScore); result.setValidatorScore(validatorScore); result.setValidatorNegativeScore(validatorNegativeScore); - result.setRootPipeLineId(action.getRootPipelineId()); + result.setRootPipelineId(action.getRootPipelineId()); result.setConfidenceScore(valConfidenceScore); result.setProcessId(String.valueOf(action.getProcessId())); result.setStatus("COMPLETED"); @@ -387,7 +387,6 @@ public static class ValidatorConfigurationDetail { private String originId; private int paperNo; private Integer groupId; - private Long rootPipeLineId; private String ProcessId; private int sorId; private int sorItemId;