Skip to content

Commit

Permalink
Merge pull request #14 from zucisystems-dev/bugs/qr_code_extraction
Browse files Browse the repository at this point in the history 
1. changes in qr code for absent qr codes in file
  • Loading branch information
@Anuradha-Oomaidurai-ZS0027
Anuradha-Oomaidurai-ZS0027 authored Apr 12, 2023
2 parents 476be59 + 21c29d6 commit f7d6d17
Show file tree
Hide file tree
Showing 3 changed files with 79 additions and 56 deletions.
2 changes: 1 addition & 1 deletion src/main/java/in/handyman/raven/lib/CoproStartAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ public void execute() throws Exception {
.ram(Optional.ofNullable(responseObj.get("cpuUtitlize")).map(String::valueOf).orElse(null))
.gpuRam(Optional.ofNullable(responseObj.get("gpuUtilize")).map(String::valueOf).orElse(null))
.build();
Thread.sleep(10000);
Thread.sleep(Long.parseLong(action.getContext().get("macro.copro.stop.action")));
jdbi.useTransaction(handle -> handle.createUpdate("INSERT INTO info.resource_utilization_details (process_id, action_id, root_pipeline_id, copro_action_name, copro_pid, created_on, command_exec, cpu, gpu, ram, gpu_ram)" +
" select :processId,:actionId,:rootPipelineId,:coproActionName,:coproProcessId,now(),:command,:cpu,:gpu,:ram,:gpuRam;")
.bindBean(coproStartActionEntity)
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/in/handyman/raven/lib/CoproStopAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ public void execute() throws Exception {
.ram(Optional.ofNullable(responseObj.get("cpuUtitlize")).map(String::valueOf).orElse(null))
.gpuRam(Optional.ofNullable(responseObj.get("gpuUtilize")).map(String::valueOf).orElse(null))
.build();
Thread.sleep(10000);
Thread.sleep(Long.parseLong(action.getContext().get("macro.copro.start.action")));
jdbi.useTransaction(handle -> handle.createUpdate("INSERT INTO info.resource_utilization_details (process_id, action_id, root_pipeline_id, copro_action_name, copro_pid, created_on, command_exec, cpu, gpu, ram, gpu_ram)" +
" select :processId,:actionId,:rootPipelineId,:coproActionName,:coproProcessId,now(),:command,:cpu,:gpu,:ram,:gpuRam;")
.bindBean(coproStopActionEntity)
Expand Down
131 changes: 77 additions & 54 deletions src/main/java/in/handyman/raven/lib/QrExtractionAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import in.handyman.raven.exception.HandymanException;
import in.handyman.raven.lambda.access.ResourceAccess;
import in.handyman.raven.lambda.action.ActionExecution;
import in.handyman.raven.lambda.action.IActionExecution;
Expand Down Expand Up @@ -55,44 +56,51 @@ public QrExtractionAction(final ActionExecutionAudit action, final Logger log,

@Override
public void execute() throws Exception {
log.info(aMarker, "<-------qr extraction Action for {} with group by eoc-id has started------->" + qrExtraction.getName());
final Jdbi jdbi = ResourceAccess.rdbmsJDBIConn(qrExtraction.getResourceConn());

//3. initiate copro processor and copro urls
final List<URL> urls = Optional.ofNullable(action.getContext().get("copro.qr-attribution.url")).map(s -> Arrays.stream(s.split(",")).map(s1 -> {
try {
return new URL(s1);
} catch (MalformedURLException e) {
log.error("Error in processing the URL ", e);
throw new RuntimeException(e);
}
}).collect(Collectors.toList())).orElse(Collections.emptyList());

log.info("Urls for the qr attribution for sor grouping {}", urls);

//5. build insert prepare statement with output table columns
final String insertQuery = "INSERT INTO " +qrExtraction.getOutputTable()+
" (origin_id, group_id, paper_no, created_on, qr_format, qr_format_id, extracted_value, file_id, b_box, angle, confidence_score, status, stage, message)" +
"VALUES(?,?,?,?, ?,?,? ,?,?, ?,?,?, ?,?)";
final CoproProcessor<CoverageInputEntity, CoverageOutputEntity> coproProcessor =
new CoproProcessor<>(new LinkedBlockingQueue<>(),
CoverageOutputEntity.class,
CoverageInputEntity.class,
jdbi, log,
new CoverageInputEntity(), urls, action);
//4. call the method start producer from coproprocessor
coproProcessor.startProducer(qrExtraction.getQuerySet(), Integer.valueOf(action.getContext().get("read.batch.size")));
log.info("start producer method from copro processor ");
Thread.sleep(1000);
//8. call the method start consumer from coproprocessor
coproProcessor.startConsumer(insertQuery, Integer.valueOf(action.getContext().get("consumer.API.count")), Integer.valueOf(action.getContext().get("write.batch.size")), new QrConsumerProcess(log, aMarker, action));
log.info("start consumer method from copro processor ");
try{
log.info(aMarker, "<-------qr extraction Action for {} with group by eoc-id has started------->" + qrExtraction.getName());
final Jdbi jdbi = ResourceAccess.rdbmsJDBIConn(qrExtraction.getResourceConn());

//3. initiate copro processor and copro urls
final List<URL> urls = Optional.ofNullable(action.getContext().get("copro.qr-attribution.url")).map(s -> Arrays.stream(s.split(",")).map(s1 -> {
try {
return new URL(s1);
} catch (MalformedURLException e) {
log.error("Error in processing the URL ", e);
throw new RuntimeException(e);
}
}).collect(Collectors.toList())).orElse(Collections.emptyList());

log.info("Urls for the qr attribution for sor grouping {}", urls);

//5. build insert prepare statement with output table columns
final String insertQuery = "INSERT INTO " +qrExtraction.getOutputTable()+
" (origin_id, group_id, paper_no, created_on, qr_format, qr_format_id, extracted_value, file_id, b_box, angle, confidence_score, status, stage, message)" +
"VALUES(?,?,?,?, ?,?,? ,?,?, ?,?,?, ?,?)";
final CoproProcessor<QrInputEntity, QrOutputEntity> coproProcessor =
new CoproProcessor<>(new LinkedBlockingQueue<>(),
QrOutputEntity.class,
QrInputEntity.class,
jdbi, log,
new QrInputEntity(), urls, action);
//4. call the method start producer from coproprocessor
coproProcessor.startProducer(qrExtraction.getQuerySet(), Integer.valueOf(action.getContext().get("read.batch.size")));
log.info("start producer method from copro processor ");
Thread.sleep(1000);
//8. call the method start consumer from coproprocessor
coproProcessor.startConsumer(insertQuery, Integer.valueOf(action.getContext().get("consumer.API.count")), Integer.valueOf(action.getContext().get("write.batch.size")), new QrConsumerProcess(log, aMarker, action));
log.info("start consumer method from copro processor ");

}catch (Exception e){
log.error("Error in the Qr extraction action ", e);
throw new HandymanException("QR extraction action failed ", e);

}

}


//6. write consumer process class which implements CoproProcessor.ConsumerProcess
public static class QrConsumerProcess implements CoproProcessor.ConsumerProcess<CoverageInputEntity, CoverageOutputEntity> {
public static class QrConsumerProcess implements CoproProcessor.ConsumerProcess<QrInputEntity, QrOutputEntity> {

private final Logger log;
private final Marker aMarker;
Expand All @@ -116,10 +124,10 @@ public QrConsumerProcess(Logger log, Marker aMarker, ActionExecutionAudit action

//7. overwrite the method process in coproprocessor, write copro api logic inside this method
@Override
public List<CoverageOutputEntity> process(URL endpoint, CoverageInputEntity entity) throws Exception {
public List<QrOutputEntity> process(URL endpoint, QrInputEntity entity) throws Exception {
log.info("copro consumer process started");

List<CoverageOutputEntity> coverageOutputEntities = new ArrayList<>();
List<QrOutputEntity> qrOutputEntities = new ArrayList<>();
final ObjectNode objectNode = mapper.createObjectNode();
objectNode.put("inputFilePath", entity.filePath);
log.info("input object node in the consumer process {}", objectNode);
Expand All @@ -129,32 +137,46 @@ public List<CoverageOutputEntity> process(URL endpoint, CoverageInputEntity enti
try (Response response = httpclient.newCall(request).execute()) {
String responseBody = Objects.requireNonNull(response.body()).string();
if (response.isSuccessful()) {
List<QrReader> donutLineItems = mapper.readValue(responseBody, new TypeReference<>() {
List<QrReader> qrLineItems = mapper.readValue(responseBody, new TypeReference<>() {
});
AtomicInteger atomicInteger = new AtomicInteger();
donutLineItems.forEach(qrReader -> {

coverageOutputEntities.add(CoverageOutputEntity.builder()
.angle(qrReader.getAngle())
AtomicInteger atomicInteger = new AtomicInteger();
if(!qrLineItems.isEmpty()){
qrLineItems.forEach(qrReader -> {

qrOutputEntities.add(QrOutputEntity.builder()
.angle(qrReader.getAngle())
.originId(entity.getOriginId())
.paperNo(entity.getPaperNo())
.groupId(entity.getGroupId())
.fileId(entity.getFileId())
.qrFormat(qrReader.getType())
.qrFormatId(atomicInteger.incrementAndGet())
.extractedValue(qrReader.getValue())
.confidenceScore(qrReader.getConfidenceScore())
.createdOn(Timestamp.valueOf(LocalDateTime.now()))
.b_box(qrReader.getBoundingBox().toString())
.status("COMPLETED")
.stage("QR_EXTRACTION")
.message("qr extraction completed")
.build());

});
}else{
qrOutputEntities.add(QrOutputEntity.builder()
.originId(entity.getOriginId())
.paperNo(entity.getPaperNo())
.groupId(entity.getGroupId())
.fileId(entity.getFileId())
.qrFormat(qrReader.getType())
.qrFormatId(atomicInteger.incrementAndGet())
.extractedValue(qrReader.getValue())
.confidenceScore(qrReader.getConfidenceScore())
.createdOn(Timestamp.valueOf(LocalDateTime.now()))
.b_box(qrReader.getBoundingBox().toString())
.status("COMPLETED")
.status("ABSENT")
.stage("QR_EXTRACTION")
.message("qr extraction completed")
.message("qr code absent in the given file")
.build());

});
}

} else {
coverageOutputEntities.add(CoverageOutputEntity.builder()
qrOutputEntities.add(QrOutputEntity.builder()
.originId(entity.getOriginId())
.paperNo(entity.getPaperNo())
.groupId(entity.getGroupId())
Expand All @@ -169,7 +191,7 @@ public List<CoverageOutputEntity> process(URL endpoint, CoverageInputEntity enti
}

} catch (Exception e) {
coverageOutputEntities.add(CoverageOutputEntity.builder()
qrOutputEntities.add(QrOutputEntity.builder()
.originId(entity.getOriginId())
.paperNo(entity.getPaperNo())
.groupId(entity.getGroupId())
Expand All @@ -180,8 +202,9 @@ public List<CoverageOutputEntity> process(URL endpoint, CoverageInputEntity enti
.message(e.toString())
.build());
log.error("Error in the copro process api hit {}", request);
throw new HandymanException("QR extraction action failed in copro processor ", e);
}
return coverageOutputEntities;
return qrOutputEntities;
}
}

Expand All @@ -203,7 +226,7 @@ public static class QrReader {
@AllArgsConstructor
@NoArgsConstructor
@Builder
public static class CoverageInputEntity implements CoproProcessor.Entity {
public static class QrInputEntity implements CoproProcessor.Entity {
private String filePath;
private Integer groupId;
private String originId;
Expand All @@ -221,7 +244,7 @@ public List<Object> getRowData() {
@AllArgsConstructor
@NoArgsConstructor
@Builder
public static class CoverageOutputEntity implements CoproProcessor.Entity {
public static class QrOutputEntity implements CoproProcessor.Entity {
private String originId;
private Integer groupId;
private Integer paperNo;
Expand Down

0 comments on commit f7d6d17

Please sign in to comment.