diff --git a/src/main/java/org/dependencytrack/tasks/metrics/ComponentMetricsUpdateTask.java b/src/main/java/org/dependencytrack/tasks/metrics/ComponentMetricsUpdateTask.java index a4eafad6da..7386749d26 100644 --- a/src/main/java/org/dependencytrack/tasks/metrics/ComponentMetricsUpdateTask.java +++ b/src/main/java/org/dependencytrack/tasks/metrics/ComponentMetricsUpdateTask.java @@ -21,6 +21,7 @@ import alpine.common.logging.Logger; import alpine.event.framework.Event; import alpine.event.framework.Subscriber; +import alpine.persistence.ScopedCustomization; import org.apache.commons.lang3.time.DurationFormatUtils; import org.dependencytrack.event.ComponentMetricsUpdateEvent; import org.dependencytrack.metrics.Metrics; @@ -176,21 +177,26 @@ static Counters updateMetrics(final UUID uuid) throws Exception { } @SuppressWarnings("unchecked") - private static List getVulnerabilities(final PersistenceManager pm, final Component component) throws Exception { + private static List getVulnerabilities(final PersistenceManager pm, final Component component) { // Using the JDO single-string syntax here because we need to pass the parameter // of the outer query (the component) to the sub-query. For some reason that does // not work with the declarative JDO API. - try (final Query query = pm.newQuery(Query.JDOQL, """ + final Query query = pm.newQuery(Query.JDOQL, """ SELECT FROM org.dependencytrack.model.Vulnerability WHERE this.components.contains(:component) && (SELECT FROM org.dependencytrack.model.Analysis a WHERE a.component == :component && a.vulnerability == this && a.suppressed == true).isEmpty() - """)) { - query.setParameters(component); - query.getFetchPlan().setGroup(Vulnerability.FetchGroup.METRICS_UPDATE.name()); + """); + query.setParameters(component); + + // NB: Set fetch group on PM level to avoid fields of the default fetch group from being loaded. + try (var ignoredPersistenceCustomization = new ScopedCustomization(pm) + .withFetchGroup(Vulnerability.FetchGroup.METRICS_UPDATE.name())) { return List.copyOf((List) query.executeList()); + } finally { + query.closeAll(); } } diff --git a/src/main/java/org/dependencytrack/tasks/metrics/PortfolioMetricsUpdateTask.java b/src/main/java/org/dependencytrack/tasks/metrics/PortfolioMetricsUpdateTask.java index 8673ba4c8c..0a4d191b53 100644 --- a/src/main/java/org/dependencytrack/tasks/metrics/PortfolioMetricsUpdateTask.java +++ b/src/main/java/org/dependencytrack/tasks/metrics/PortfolioMetricsUpdateTask.java @@ -22,6 +22,7 @@ import alpine.common.util.SystemUtil; import alpine.event.framework.Event; import alpine.event.framework.Subscriber; +import alpine.persistence.ScopedCustomization; import org.apache.commons.lang3.time.DurationFormatUtils; import org.dependencytrack.event.CallbackEvent; import org.dependencytrack.event.PortfolioMetricsUpdateEvent; @@ -68,11 +69,11 @@ private void updateMetrics() throws Exception { final PersistenceManager pm = qm.getPersistenceManager(); LOGGER.debug("Fetching first " + BATCH_SIZE + " projects"); - List activeProjects = fetchNextActiveProjectsPage(pm, null); + List activeProjects = fetchNextActiveProjectsBatch(pm, null); while (!activeProjects.isEmpty()) { - final long firstId = activeProjects.get(0).getId(); - final long lastId = activeProjects.get(activeProjects.size() - 1).getId(); + final long firstId = activeProjects.getFirst().getId(); + final long lastId = activeProjects.getLast().getId(); final int batchCount = activeProjects.size(); final var countDownLatch = new CountDownLatch(batchCount); @@ -113,7 +114,7 @@ private void updateMetrics() throws Exception { counters.medium += metrics.getMedium(); counters.low += metrics.getLow(); counters.unassigned += metrics.getUnassigned(); - counters.vulnerabilities += metrics.getVulnerabilities(); + counters.vulnerabilities += Math.toIntExact(metrics.getVulnerabilities()); counters.findingsTotal += metrics.getFindingsTotal(); counters.findingsAudited += metrics.getFindingsAudited(); @@ -145,8 +146,13 @@ private void updateMetrics() throws Exception { counters.policyViolationsOperationalUnaudited += metrics.getPolicyViolationsOperationalUnaudited(); } + // Remove projects and project metrics from the L1 cache + // to prevent it from growing too large. + pm.evictAll(false, Project.class); + pm.evictAll(false, ProjectMetrics.class); + LOGGER.debug("Fetching next " + BATCH_SIZE + " projects"); - activeProjects = fetchNextActiveProjectsPage(pm, lastId); + activeProjects = fetchNextActiveProjectsBatch(pm, lastId); } qm.runInTransaction(() -> { @@ -166,18 +172,23 @@ private void updateMetrics() throws Exception { DurationFormatUtils.formatDuration(new Date().getTime() - counters.measuredAt.getTime(), "mm:ss:SS")); } - private List fetchNextActiveProjectsPage(final PersistenceManager pm, final Long lastId) throws Exception { - try (final Query query = pm.newQuery(Project.class)) { - if (lastId == null) { - query.setFilter("(active == null || active == true)"); - } else { - query.setFilter("(active == null || active == true) && id < :lastId"); - query.setParameters(lastId); - } - query.setOrdering("id DESC"); - query.range(0, BATCH_SIZE); - query.getFetchPlan().setGroup(Project.FetchGroup.METRICS_UPDATE.name()); + private List fetchNextActiveProjectsBatch(final PersistenceManager pm, final Long lastId) { + final Query query = pm.newQuery(Project.class); + if (lastId == null) { + query.setFilter("(active == null || active == true)"); + } else { + query.setFilter("(active == null || active == true) && id < :lastId"); + query.setParameters(lastId); + } + query.setOrdering("id DESC"); + query.range(0, BATCH_SIZE); + + // NB: Set fetch group on PM level to avoid fields of the default fetch group from being loaded. + try (var ignoredPersistenceCustomization = new ScopedCustomization(pm) + .withFetchGroup(Project.FetchGroup.METRICS_UPDATE.name())) { return List.copyOf(query.executeList()); + } finally { + query.closeAll(); } } diff --git a/src/main/java/org/dependencytrack/tasks/metrics/ProjectMetricsUpdateTask.java b/src/main/java/org/dependencytrack/tasks/metrics/ProjectMetricsUpdateTask.java index a42cd9bf1c..6964560a52 100644 --- a/src/main/java/org/dependencytrack/tasks/metrics/ProjectMetricsUpdateTask.java +++ b/src/main/java/org/dependencytrack/tasks/metrics/ProjectMetricsUpdateTask.java @@ -21,6 +21,7 @@ import alpine.common.logging.Logger; import alpine.event.framework.Event; import alpine.event.framework.Subscriber; +import alpine.persistence.ScopedCustomization; import org.apache.commons.lang3.time.DurationFormatUtils; import org.dependencytrack.event.ProjectMetricsUpdateEvent; import org.dependencytrack.metrics.Metrics; @@ -73,6 +74,8 @@ private void updateMetrics(final UUID uuid) throws Exception { List components = fetchNextComponentsPage(pm, project, null); while (!components.isEmpty()) { + final long lastId = components.getLast().getId(); + for (final Component component : components) { final Counters componentCounters; try { @@ -123,8 +126,12 @@ private void updateMetrics(final UUID uuid) throws Exception { counters.policyViolationsOperationalUnaudited += componentCounters.policyViolationsOperationalUnaudited; } + // Remove components from the L1 cache to prevent it from growing too large. + // Note that because ComponentMetricsUpdateTask uses its own QueryManager, + // component metrics objects are not in this L1 cache. + pm.evictAll(false, Component.class); + LOGGER.debug("Fetching next components page for project " + uuid); - final long lastId = components.get(components.size() - 1).getId(); components = fetchNextComponentsPage(pm, project, lastId); } @@ -141,7 +148,7 @@ private void updateMetrics(final UUID uuid) throws Exception { }); if (project.getLastInheritedRiskScore() == null || - project.getLastInheritedRiskScore() != counters.inheritedRiskScore) { + project.getLastInheritedRiskScore() != counters.inheritedRiskScore) { LOGGER.debug("Updating inherited risk score of project " + uuid); qm.runInTransaction(() -> project.setLastInheritedRiskScore(counters.inheritedRiskScore)); } @@ -151,19 +158,24 @@ private void updateMetrics(final UUID uuid) throws Exception { DurationFormatUtils.formatDuration(new Date().getTime() - counters.measuredAt.getTime(), "mm:ss:SS")); } - private List fetchNextComponentsPage(final PersistenceManager pm, final Project project, final Long lastId) throws Exception { - try (final Query query = pm.newQuery(Component.class)) { - if (lastId == null) { - query.setFilter("project == :project"); - query.setParameters(project); - } else { - query.setFilter("project == :project && id < :lastId"); - query.setParameters(project, lastId); - } - query.setOrdering("id DESC"); - query.setRange(0, 500); - query.getFetchPlan().setGroup(Component.FetchGroup.METRICS_UPDATE.name()); + private List fetchNextComponentsPage(final PersistenceManager pm, final Project project, final Long lastId) { + final Query query = pm.newQuery(Component.class); + if (lastId == null) { + query.setFilter("project == :project"); + query.setParameters(project); + } else { + query.setFilter("project == :project && id < :lastId"); + query.setParameters(project, lastId); + } + query.setOrdering("id DESC"); + query.setRange(0, 1000); + + // NB: Set fetch group on PM level to avoid fields of the default fetch group from being loaded. + try (var ignoredPersistenceCustomization = new ScopedCustomization(pm) + .withFetchGroup(Component.FetchGroup.METRICS_UPDATE.name())) { return List.copyOf(query.executeList()); + } finally { + query.closeAll(); } }