From c4babe55f63cfe8db8a17179d75faee31c35f626 Mon Sep 17 00:00:00 2001 From: Kevin Day Date: Mon, 22 Apr 2024 16:01:33 -0500 Subject: [PATCH] Relocate "generate" CLI code outside of transaction and improve error handling. The relocation allows for faster transactional operation. With the loop outside of the transaction, then the transaction can be committed for each pass. This then allows for reduced memory requirements for large transactions (such as when generate a huge amount of submissions). The exception handling is improved by catching the exceptions at the CLI level such that exceptions can be reported without crashing the entire server. Now, if a CLI fails due to some error, then the server does not exit. --- src/main/java/org/tdl/vireo/cli/Cli.java | 171 +++++++------ .../org/tdl/vireo/service/CliService.java | 228 +++++++++--------- 2 files changed, 209 insertions(+), 190 deletions(-) diff --git a/src/main/java/org/tdl/vireo/cli/Cli.java b/src/main/java/org/tdl/vireo/cli/Cli.java index 4f714e623..e0b5b207b 100644 --- a/src/main/java/org/tdl/vireo/cli/Cli.java +++ b/src/main/java/org/tdl/vireo/cli/Cli.java @@ -2,12 +2,14 @@ import java.util.ArrayList; import java.util.List; +import java.util.Random; import java.util.Scanner; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.CommandLineRunner; import org.springframework.core.Ordered; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; +import org.tdl.vireo.model.User; import org.tdl.vireo.service.CliService; /** @@ -23,7 +25,7 @@ @Order(value = Ordered.LOWEST_PRECEDENCE) @Component public class Cli implements CommandLineRunner { - + @Autowired CliService cliService; @@ -40,106 +42,129 @@ public void run(String... args) throws Exception { Boolean running = runConsole ? true : false; if (running) { + final Scanner reader = new Scanner(System.in); // Reading from System.in final String PROMPT = "\n" + (char) 27 + "[36mvireo>" + (char) 27 + "[37m "; - - Scanner reader = new Scanner(System.in); // Reading from System.in boolean expansive = false; int itemsGenerated = 0; System.out.print(PROMPT); while (running && reader.hasNextLine()) { + try { + String n = reader.nextLine(); - String n = reader.nextLine(); + n = n.trim(); - n = n.trim(); + String[] commandTokens = n.split("\\s+"); + List commandArgs = new ArrayList(); - String[] commandTokens = n.split("\\s+"); - List commandArgs = new ArrayList(); + String command = null; + int num1 = 0; + int num2 = 0; - String command = null; - int num1 = 0; - int num2 = 0; + if (commandTokens.length > 0) + command = commandTokens[0]; + if (commandTokens.length > 1) { + for (int i = 1; i < commandTokens.length; i++) { + commandArgs.add(commandTokens[i]); + } - if (commandTokens.length > 0) - command = commandTokens[0]; - if (commandTokens.length > 1) { - for (int i = 1; i < commandTokens.length; i++) { - commandArgs.add(commandTokens[i]); } - } - - switch (command) { - case "exit": - System.out.println("\nGoodbye."); - running = false; - break; - - case "accounts": - num1 = 0; - if (commandArgs.size() > 0) { - try { - num1 = Integer.parseInt(commandArgs.get(0)); - } catch (Exception e) { - System.err.println("unable to parse as a number of items: " + commandArgs.get(0)); + switch (command) { + case "exit": + System.out.println("\nGoodbye."); + running = false; + break; + + case "accounts": + num1 = 0; + if (commandArgs.size() > 0) { + try { + num1 = Integer.parseInt(commandArgs.get(0)); + } catch (Exception e) { + System.err.println("unable to parse as a number of items: " + commandArgs.get(0)); + } } - } - cliService.operateAccounts(expansive, num1); - break; + cliService.operateAccounts(expansive, num1); + break; - case "admin_accounts": - num1 = 0; - if (commandArgs.size() > 0) { - try { - num1 = Integer.parseInt(commandArgs.get(0)); - } catch (Exception e) { - System.err.println("unable to parse as a number of items: " + commandArgs.get(0)); + case "admin_accounts": + num1 = 0; + if (commandArgs.size() > 0) { + try { + num1 = Integer.parseInt(commandArgs.get(0)); + } catch (Exception e) { + System.err.println("unable to parse as a number of items: " + commandArgs.get(0)); + } } - } - cliService.operateAdminAccounts(expansive, num1); - break; - - case "expansive": - expansive = !expansive; - System.out.println("\nSet expansive = " + (expansive ? "true" : "false") + "."); - break; - - case "generate": - num1 = 0; - num2 = 100; - if (commandArgs.size() > 0) { - try { - // First argument is number of submissions. - num1 = Integer.parseInt(commandArgs.get(0)); - - // Second argument is maximum action logs. - if (commandArgs.size() > 1) { - num2 = Integer.parseInt(commandArgs.get(1)); + cliService.operateAdminAccounts(expansive, num1); + break; + + case "expansive": + expansive = !expansive; + System.out.println("\nSet expansive = " + (expansive ? "true" : "false") + "."); + break; + + case "generate": + num1 = 0; + num2 = 100; + if (commandArgs.size() > 0) { + try { + // First argument is number of submissions. + num1 = Integer.parseInt(commandArgs.get(0)); + + // Second argument is maximum action logs. + if (commandArgs.size() > 1) { + num2 = Integer.parseInt(commandArgs.get(1)); + } + } catch (Exception e) { + System.err.println("unable to parse as a number of items: " + commandArgs.get(0)); } - } catch (Exception e) { - System.err.println("unable to parse as a number of items: " + commandArgs.get(0)); } - } - cliService.operateGenerate(expansive, num1, num2, itemsGenerated); - break; + { + Random random = new Random(); + int idOffset = cliService.countUsers(); + User helpfulHarry = null; - case "": - break; + if (expansive) { + helpfulHarry = cliService.createHelpfulHarry(idOffset++, CliService.EMAIL_DATE); + } - default: - System.out.println("Unknown command " + n); - } + for (int i = itemsGenerated; i < num1 + itemsGenerated; i++) { + cliService.operateGenerate(expansive, num2, random, idOffset, helpfulHarry, i); + System.out.print("\r" + (i - itemsGenerated) + " of " + num1 + " generated..."); + } + + if (expansive) { + System.out.println("\rGenerated " + num1 + " submissions expansively with max action logs of " + num2 + "."); + } else { + System.out.println("\rGenerated " + num1 + " submissions."); + } + + itemsGenerated += num1; + } + + break; - if (running) - System.out.print(PROMPT); + case "": + break; + default: + System.out.println("Unknown command " + n); + } + + if (running) System.out.print(PROMPT); + + } catch (Exception e) { + e.printStackTrace(); + } } - reader.close(); + reader.close(); } } diff --git a/src/main/java/org/tdl/vireo/service/CliService.java b/src/main/java/org/tdl/vireo/service/CliService.java index 5daa6eeb5..75fcbb732 100644 --- a/src/main/java/org/tdl/vireo/service/CliService.java +++ b/src/main/java/org/tdl/vireo/service/CliService.java @@ -33,6 +33,12 @@ @Service public class CliService { + public static final SimpleDateFormat FORMAT_DAY = new SimpleDateFormat("yyyy-MM-dd"); + + public static final SimpleDateFormat FORMAT_MONTH = new SimpleDateFormat("MMMM yyyy"); + + public static final SimpleDateFormat EMAIL_DATE = new SimpleDateFormat("_yyyy_MM_dd_HH_mm_ss_"); + @Autowired private UserRepo userRepo; @@ -81,139 +87,132 @@ public void operateAdminAccounts(boolean expansive, int generateTotal) { } } - public void operateGenerate(boolean expansive, int generateTotal, int maxActionLogs, int itemsGenerated) throws OrganizationDoesNotAcceptSubmissionsException { - Random random = new Random(); - Calendar calendar = null; - - SimpleDateFormat formatDay = new SimpleDateFormat("yyyy-MM-dd"); - SimpleDateFormat formatMonth = new SimpleDateFormat("MMMM yyyy"); - SimpleDateFormat emailDate = new SimpleDateFormat("_yyyy_MM_dd_HH_mm_ss_"); + public void operateGenerate(boolean expansive, int maxActionLogs, Random random, int idOffset, User helpfulHarry, int i) throws OrganizationDoesNotAcceptSubmissionsException { + Calendar now = Calendar.getInstance(); + User submitter = userRepo.create("bob" + EMAIL_DATE.format(now.getTime()) + (idOffset + i + 1) + "@boring.bob", "bob", "boring " + (idOffset + i + 1), Role.ROLE_STUDENT); + Credentials credentials = new Credentials(); + credentials.setFirstName("Bob"); + credentials.setLastName("Boring " + (idOffset + i + 1)); + credentials.setEmail("bob@boring.bob"); + credentials.setRole(Role.ROLE_STUDENT.name()); - Organization org = organizationRepo.findAll().get(0); + Organization org = getOrganization(0); + setAcceptSubmissions(org); - int idOffset = userRepo.findAll().toArray().length; - User helpfulHarry = null; + List statuses = getAllSubmissionStatuses(); + SubmissionStatus state = statuses.get(0); + // Status is chosen completely randomly for every option available when expansive is enabled. if (expansive) { - helpfulHarry = createHelpfulHarry(idOffset++, emailDate); + state = statuses.get(random.nextInt(statuses.size())); } - if (!org.getAcceptsSubmissions()) { - org.setAcceptsSubmissions(true); - org = organizationRepo.save(org); - } - - List statuses = submissionStatusRepo.findAll(); - - for (int i = itemsGenerated; i < generateTotal + itemsGenerated; i++) { - Calendar now = Calendar.getInstance(); - User submitter = userRepo.create("bob" + emailDate.format(now.getTime()) + (idOffset + i + 1) + "@boring.bob", "bob", "boring " + (idOffset + i + 1), Role.ROLE_STUDENT); - Credentials credentials = new Credentials(); - credentials.setFirstName("Bob"); - credentials.setLastName("Boring " + (idOffset + i + 1)); - credentials.setEmail("bob@boring.bob"); - credentials.setRole(Role.ROLE_STUDENT.name()); - - SubmissionStatus state = statuses.get(0); - - // Status is chosen completely randomly for every option available when expansive is enabled. - if (expansive) { - state = statuses.get(random.nextInt(statuses.size())); - } - - List customActions = customActionDefinitionRepo.findAll(); + List customActions = customActionDefinitionRepo.findAll(); - Submission sub = submissionRepo.create(submitter, org, state, credentials, customActions); + Submission sub = submissionRepo.create(submitter, org, state, credentials, customActions); - sub.setSubmissionDate(getRandomDate()); + sub.setSubmissionDate(getRandomDate()); - if (random.nextInt(10) < 3) { - sub.setApproveAdvisorDate(getRandomDate()); - sub.setApproveAdvisor(random.nextInt(10) < 5); - } else { - sub.setApproveAdvisorDate(null); - sub.setApproveAdvisor(false); - } + if (random.nextInt(10) < 3) { + sub.setApproveAdvisorDate(getRandomDate()); + sub.setApproveAdvisor(random.nextInt(10) < 5); + } else { + sub.setApproveAdvisorDate(null); + sub.setApproveAdvisor(false); + } - if (random.nextInt(10) < 3) { - sub.setApproveApplicationDate(getRandomDate()); - sub.setApproveApplication(random.nextInt(10) < 5); - } else { - sub.setApproveApplicationDate(null); - sub.setApproveApplication(false); - } + if (random.nextInt(10) < 3) { + sub.setApproveApplicationDate(getRandomDate()); + sub.setApproveApplication(random.nextInt(10) < 5); + } else { + sub.setApproveApplicationDate(null); + sub.setApproveApplication(false); + } - if (random.nextInt(10) < 3) { - sub.setApproveEmbargoDate(getRandomDate()); - sub.setApproveEmbargo(random.nextInt(10) < 5); - } else { - sub.setApproveEmbargoDate(null); - sub.setApproveEmbargo(false); - } + if (random.nextInt(10) < 3) { + sub.setApproveEmbargoDate(getRandomDate()); + sub.setApproveEmbargo(random.nextInt(10) < 5); + } else { + sub.setApproveEmbargoDate(null); + sub.setApproveEmbargo(false); + } - // %30 chance to be assigned to helpful harry. - if (expansive && random.nextInt(10) < 3) { - sub.setAssignee(helpfulHarry); - } + // 30% chance to be assigned to helpful harry. + if (expansive && random.nextInt(10) < 3) { + sub.setAssignee(helpfulHarry); + } - generateActionLogs(sub, submitter, expansive, maxActionLogs); - - for (SubmissionWorkflowStep step : sub.getSubmissionWorkflowSteps()) { - for (SubmissionFieldProfile fp : step.getAggregateFieldProfiles()) { - FieldPredicate pred = fp.getFieldPredicate(); - FieldValue val; - switch (fp.getInputType().getName()) { - case "INPUT_FILE": - break; - case "INPUT_CONTACT": - val = fieldValueRepo.create(pred); + generateActionLogs(sub, submitter, expansive, maxActionLogs); + + for (SubmissionWorkflowStep step : sub.getSubmissionWorkflowSteps()) { + for (SubmissionFieldProfile fp : step.getAggregateFieldProfiles()) { + FieldPredicate pred = fp.getFieldPredicate(); + FieldValue val; + + switch (fp.getInputType().getName()) { + case "INPUT_FILE": + break; + case "INPUT_CONTACT": + val = fieldValueRepo.create(pred); + val.setValue("test " + pred.getValue() + " " + i); + val.setContacts(Arrays.asList(new String[] { "test" + pred.getValue() + i + "@mailinator.com" })); + sub.addFieldValue(val); + break; + case "INPUT_EMAIL": + val = fieldValueRepo.create(pred); + val.setValue("test" + pred.getValue() + i + "@mailinator.com"); + sub.addFieldValue(val); + break; + case "INPUT_DEGREEDATE": + val = fieldValueRepo.create(pred); + val.setValue(FORMAT_MONTH.format(getRandomDegreeDate().getTime())); + sub.addFieldValue(val); + break; + case "INPUT_DATE": + val = fieldValueRepo.create(pred); + val.setValue(FORMAT_DAY.format(getRandomDate().getTime())); + sub.addFieldValue(val); + break; + default: + // Allow for a small number of unfilled field values. + if (random.nextInt(10) > 8) break; + + val = fieldValueRepo.create(pred); + if (pred.getValue().equalsIgnoreCase("birth_year")) { + val.setValue(getRandomYearString(80)); + } else { val.setValue("test " + pred.getValue() + " " + i); - val.setContacts(Arrays.asList(new String[] { "test" + pred.getValue() + i + "@mailinator.com" })); - sub.addFieldValue(val); - break; - case "INPUT_EMAIL": - val = fieldValueRepo.create(pred); - val.setValue("test" + pred.getValue() + i + "@mailinator.com"); - sub.addFieldValue(val); - break; - case "INPUT_DEGREEDATE": - val = fieldValueRepo.create(pred); - calendar = getRandomDegreeDate(); - val.setValue(formatMonth.format(calendar.getTime())); - sub.addFieldValue(val); - break; - case "INPUT_DATE": - val = fieldValueRepo.create(pred); - calendar = getRandomDate(); - val.setValue(formatDay.format(calendar.getTime())); - sub.addFieldValue(val); - break; - default: - // Allow for a small number of unfilled field values. - if (random.nextInt(10) > 8) break; - - val = fieldValueRepo.create(pred); - if (pred.getValue().equalsIgnoreCase("birth_year")) { - val.setValue(getRandomYearString(80)); - } else { - val.setValue("test " + pred.getValue() + " " + i); - } - sub.addFieldValue(val); } + sub.addFieldValue(val); } } - submissionRepo.saveAndFlush(sub); - - System.out.print("\r" + (i - itemsGenerated) + " of " + generateTotal + " generated..."); } - if (expansive) { - System.out.println("\rGenerated " + generateTotal + " submissions expansively with max action logs of " + maxActionLogs + "."); - } else { - System.out.println("\rGenerated " + generateTotal + " submissions."); + submissionRepo.saveAndFlush(sub); + } + + public Organization getOrganization(int index) { + return organizationRepo.findAll().get(index); + } + + public int countUsers() { + return userRepo.findAll().toArray().length; + } + + public void setAcceptSubmissions(Organization organization) { + if (!organization.getAcceptsSubmissions()) { + organization.setAcceptsSubmissions(true); + organization = organizationRepo.save(organization); } + } - itemsGenerated += generateTotal; + public List getAllSubmissionStatuses() { + return submissionStatusRepo.findAll(); + } + + public User createHelpfulHarry(int offset, SimpleDateFormat formatter) { + String dateString = formatter.format(Calendar.getInstance().getTime()); + return userRepo.create("harry" + dateString + offset + "@help.ful", "Harry", "Helpful " + offset, Role.ROLE_REVIEWER); } private Calendar getRandomDate() { @@ -257,11 +256,6 @@ private String getRandomYearString(int max) { return "" + (date.get(Calendar.YEAR) - random.nextInt(max)); } - private User createHelpfulHarry(int offset, SimpleDateFormat formatter) { - String dateString = formatter.format(Calendar.getInstance().getTime()); - return userRepo.create("harry" + dateString + offset + "@help.ful", "Harry", "Helpful " + offset, Role.ROLE_REVIEWER); - } - private void generateActionLogs(Submission sub, User submitter, boolean expansive, int maxActionLogs) { actionLogRepo.create(sub, submitter, Calendar.getInstance(), new String("Submission created."), false);