diff --git a/src/main/java/org/dependencytrack/tasks/metrics/ComponentMetricsUpdateTask.java b/src/main/java/org/dependencytrack/tasks/metrics/ComponentMetricsUpdateTask.java
index a4eafad6da..7386749d26 100644
--- a/src/main/java/org/dependencytrack/tasks/metrics/ComponentMetricsUpdateTask.java
+++ b/src/main/java/org/dependencytrack/tasks/metrics/ComponentMetricsUpdateTask.java
@@ -21,6 +21,7 @@
import alpine.common.logging.Logger;
import alpine.event.framework.Event;
import alpine.event.framework.Subscriber;
+import alpine.persistence.ScopedCustomization;
import org.apache.commons.lang3.time.DurationFormatUtils;
import org.dependencytrack.event.ComponentMetricsUpdateEvent;
import org.dependencytrack.metrics.Metrics;
@@ -176,21 +177,26 @@ static Counters updateMetrics(final UUID uuid) throws Exception {
}
@SuppressWarnings("unchecked")
- private static List<Vulnerability> getVulnerabilities(final PersistenceManager pm, final Component component) throws Exception {
+ private static List<Vulnerability> getVulnerabilities(final PersistenceManager pm, final Component component) {
// Using the JDO single-string syntax here because we need to pass the parameter
// of the outer query (the component) to the sub-query. For some reason that does
// not work with the declarative JDO API.
- try (final Query<?> query = pm.newQuery(Query.JDOQL, """
+ final Query<?> query = pm.newQuery(Query.JDOQL, """
SELECT FROM org.dependencytrack.model.Vulnerability
WHERE this.components.contains(:component)
&& (SELECT FROM org.dependencytrack.model.Analysis a
WHERE a.component == :component
&& a.vulnerability == this
&& a.suppressed == true).isEmpty()
- """)) {
- query.setParameters(component);
- query.getFetchPlan().setGroup(Vulnerability.FetchGroup.METRICS_UPDATE.name());
+ """);
+ query.setParameters(component);
+
+ // NB: Set fetch group on PM level to avoid fields of the default fetch group from being loaded.
+ try (var ignoredPersistenceCustomization = new ScopedCustomization(pm)
+ .withFetchGroup(Vulnerability.FetchGroup.METRICS_UPDATE.name())) {
return List.copyOf((List<Vulnerability>) query.executeList());
+ } finally {
+ query.closeAll();
}
}
diff --git a/src/main/java/org/dependencytrack/tasks/metrics/PortfolioMetricsUpdateTask.java b/src/main/java/org/dependencytrack/tasks/metrics/PortfolioMetricsUpdateTask.java
index 8673ba4c8c..0a4d191b53 100644
--- a/src/main/java/org/dependencytrack/tasks/metrics/PortfolioMetricsUpdateTask.java
+++ b/src/main/java/org/dependencytrack/tasks/metrics/PortfolioMetricsUpdateTask.java
@@ -22,6 +22,7 @@
import alpine.common.util.SystemUtil;
import alpine.event.framework.Event;
import alpine.event.framework.Subscriber;
+import alpine.persistence.ScopedCustomization;
import org.apache.commons.lang3.time.DurationFormatUtils;
import org.dependencytrack.event.CallbackEvent;
import org.dependencytrack.event.PortfolioMetricsUpdateEvent;
@@ -68,11 +69,11 @@ private void updateMetrics() throws Exception {
final PersistenceManager pm = qm.getPersistenceManager();
LOGGER.debug("Fetching first " + BATCH_SIZE + " projects");
- List<Project> activeProjects = fetchNextActiveProjectsPage(pm, null);
+ List<Project> activeProjects = fetchNextActiveProjectsBatch(pm, null);
while (!activeProjects.isEmpty()) {
- final long firstId = activeProjects.get(0).getId();
- final long lastId = activeProjects.get(activeProjects.size() - 1).getId();
+ final long firstId = activeProjects.getFirst().getId();
+ final long lastId = activeProjects.getLast().getId();
final int batchCount = activeProjects.size();
final var countDownLatch = new CountDownLatch(batchCount);
@@ -113,7 +114,7 @@ private void updateMetrics() throws Exception {
counters.medium += metrics.getMedium();
counters.low += metrics.getLow();
counters.unassigned += metrics.getUnassigned();
- counters.vulnerabilities += metrics.getVulnerabilities();
+ counters.vulnerabilities += Math.toIntExact(metrics.getVulnerabilities());
counters.findingsTotal += metrics.getFindingsTotal();
counters.findingsAudited += metrics.getFindingsAudited();
@@ -145,8 +146,13 @@ private void updateMetrics() throws Exception {
counters.policyViolationsOperationalUnaudited += metrics.getPolicyViolationsOperationalUnaudited();
}
+ // Remove projects and project metrics from the L1 cache
+ // to prevent it from growing too large.
+ pm.evictAll(false, Project.class);
+ pm.evictAll(false, ProjectMetrics.class);
+
LOGGER.debug("Fetching next " + BATCH_SIZE + " projects");
- activeProjects = fetchNextActiveProjectsPage(pm, lastId);
+ activeProjects = fetchNextActiveProjectsBatch(pm, lastId);
}
qm.runInTransaction(() -> {
@@ -166,18 +172,23 @@ private void updateMetrics() throws Exception {
DurationFormatUtils.formatDuration(new Date().getTime() - counters.measuredAt.getTime(), "mm:ss:SS"));
}
- private List<Project> fetchNextActiveProjectsPage(final PersistenceManager pm, final Long lastId) throws Exception {
- try (final Query<Project> query = pm.newQuery(Project.class)) {
- if (lastId == null) {
- query.setFilter("(active == null || active == true)");
- } else {
- query.setFilter("(active == null || active == true) && id < :lastId");
- query.setParameters(lastId);
- }
- query.setOrdering("id DESC");
- query.range(0, BATCH_SIZE);
- query.getFetchPlan().setGroup(Project.FetchGroup.METRICS_UPDATE.name());
+ private List<Project> fetchNextActiveProjectsBatch(final PersistenceManager pm, final Long lastId) {
+ final Query<Project> query = pm.newQuery(Project.class);
+ if (lastId == null) {
+ query.setFilter("(active == null || active == true)");
+ } else {
+ query.setFilter("(active == null || active == true) && id < :lastId");
+ query.setParameters(lastId);
+ }
+ query.setOrdering("id DESC");
+ query.range(0, BATCH_SIZE);
+
+ // NB: Set fetch group on PM level to avoid fields of the default fetch group from being loaded.
+ try (var ignoredPersistenceCustomization = new ScopedCustomization(pm)
+ .withFetchGroup(Project.FetchGroup.METRICS_UPDATE.name())) {
return List.copyOf(query.executeList());
+ } finally {
+ query.closeAll();
}
}
diff --git a/src/main/java/org/dependencytrack/tasks/metrics/ProjectMetricsUpdateTask.java b/src/main/java/org/dependencytrack/tasks/metrics/ProjectMetricsUpdateTask.java
index a42cd9bf1c..6964560a52 100644
--- a/src/main/java/org/dependencytrack/tasks/metrics/ProjectMetricsUpdateTask.java
+++ b/src/main/java/org/dependencytrack/tasks/metrics/ProjectMetricsUpdateTask.java
@@ -21,6 +21,7 @@
import alpine.common.logging.Logger;
import alpine.event.framework.Event;
import alpine.event.framework.Subscriber;
+import alpine.persistence.ScopedCustomization;
import org.apache.commons.lang3.time.DurationFormatUtils;
import org.dependencytrack.event.ProjectMetricsUpdateEvent;
import org.dependencytrack.metrics.Metrics;
@@ -73,6 +74,8 @@ private void updateMetrics(final UUID uuid) throws Exception {
List<Component> components = fetchNextComponentsPage(pm, project, null);
while (!components.isEmpty()) {
+ final long lastId = components.getLast().getId();
+
for (final Component component : components) {
final Counters componentCounters;
try {
@@ -123,8 +126,12 @@ private void updateMetrics(final UUID uuid) throws Exception {
counters.policyViolationsOperationalUnaudited += componentCounters.policyViolationsOperationalUnaudited;
}
+ // Remove components from the L1 cache to prevent it from growing too large.
+ // Note that because ComponentMetricsUpdateTask uses its own QueryManager,
+ // component metrics objects are not in this L1 cache.
+ pm.evictAll(false, Component.class);
+
LOGGER.debug("Fetching next components page for project " + uuid);
- final long lastId = components.get(components.size() - 1).getId();
components = fetchNextComponentsPage(pm, project, lastId);
}
@@ -141,7 +148,7 @@ private void updateMetrics(final UUID uuid) throws Exception {
});
if (project.getLastInheritedRiskScore() == null ||
- project.getLastInheritedRiskScore() != counters.inheritedRiskScore) {
+ project.getLastInheritedRiskScore() != counters.inheritedRiskScore) {
LOGGER.debug("Updating inherited risk score of project " + uuid);
qm.runInTransaction(() -> project.setLastInheritedRiskScore(counters.inheritedRiskScore));
}
@@ -151,19 +158,24 @@ private void updateMetrics(final UUID uuid) throws Exception {
DurationFormatUtils.formatDuration(new Date().getTime() - counters.measuredAt.getTime(), "mm:ss:SS"));
}
- private List<Component> fetchNextComponentsPage(final PersistenceManager pm, final Project project, final Long lastId) throws Exception {
- try (final Query<Component> query = pm.newQuery(Component.class)) {
- if (lastId == null) {
- query.setFilter("project == :project");
- query.setParameters(project);
- } else {
- query.setFilter("project == :project && id < :lastId");
- query.setParameters(project, lastId);
- }
- query.setOrdering("id DESC");
- query.setRange(0, 500);
- query.getFetchPlan().setGroup(Component.FetchGroup.METRICS_UPDATE.name());
+ private List<Component> fetchNextComponentsPage(final PersistenceManager pm, final Project project, final Long lastId) {
+ final Query<Component> query = pm.newQuery(Component.class);
+ if (lastId == null) {
+ query.setFilter("project == :project");
+ query.setParameters(project);
+ } else {
+ query.setFilter("project == :project && id < :lastId");
+ query.setParameters(project, lastId);
+ }
+ query.setOrdering("id DESC");
+ query.setRange(0, 1000);
+
+ // NB: Set fetch group on PM level to avoid fields of the default fetch group from being loaded.
+ try (var ignoredPersistenceCustomization = new ScopedCustomization(pm)
+ .withFetchGroup(Component.FetchGroup.METRICS_UPDATE.name())) {
return List.copyOf(query.executeList());
+ } finally {
+ query.closeAll();
}
}