Skip to content

Commit

Permalink
Reduce memory usage of metrics update tasks
Browse files Browse the repository at this point in the history
Because the tasks iterated over large datasets (i.e. all projects, or all components of a project), the DataNucleus L1 cache kept growing. Prevent this from happening by evicting objects from the cache after every iteration.

Also, because setting a `FetchGroup` on `Query` level doesn't overwrite the `FetchGroup` on `PersistenceManager` level, too many fields were being loaded. Use `ScopedCustomization` to overwrite the PM's `FetchGroup` when querying objects for metrics calculation.

Signed-off-by: nscuro <[email protected]>
  • Loading branch information
nscuro committed Nov 14, 2024
1 parent 25a2524 commit d2adc7e
Show file tree
Hide file tree
Showing 3 changed files with 64 additions and 35 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import alpine.common.logging.Logger;
import alpine.event.framework.Event;
import alpine.event.framework.Subscriber;
import alpine.persistence.ScopedCustomization;
import org.apache.commons.lang3.time.DurationFormatUtils;
import org.dependencytrack.event.ComponentMetricsUpdateEvent;
import org.dependencytrack.metrics.Metrics;
Expand Down Expand Up @@ -176,21 +177,26 @@ static Counters updateMetrics(final UUID uuid) throws Exception {
}

@SuppressWarnings("unchecked")
private static List<Vulnerability> getVulnerabilities(final PersistenceManager pm, final Component component) throws Exception {
private static List<Vulnerability> getVulnerabilities(final PersistenceManager pm, final Component component) {
// Using the JDO single-string syntax here because we need to pass the parameter
// of the outer query (the component) to the sub-query. For some reason that does
// not work with the declarative JDO API.
try (final Query<?> query = pm.newQuery(Query.JDOQL, """
final Query<?> query = pm.newQuery(Query.JDOQL, """
SELECT FROM org.dependencytrack.model.Vulnerability
WHERE this.components.contains(:component)
&& (SELECT FROM org.dependencytrack.model.Analysis a
WHERE a.component == :component
&& a.vulnerability == this
&& a.suppressed == true).isEmpty()
""")) {
query.setParameters(component);
query.getFetchPlan().setGroup(Vulnerability.FetchGroup.METRICS_UPDATE.name());
""");
query.setParameters(component);

// NB: Set fetch group on PM level to avoid fields of the default fetch group from being loaded.
try (var ignoredPersistenceCustomization = new ScopedCustomization(pm)
.withFetchGroup(Vulnerability.FetchGroup.METRICS_UPDATE.name())) {
return List.copyOf((List<Vulnerability>) query.executeList());
} finally {
query.closeAll();
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import alpine.common.util.SystemUtil;
import alpine.event.framework.Event;
import alpine.event.framework.Subscriber;
import alpine.persistence.ScopedCustomization;
import org.apache.commons.lang3.time.DurationFormatUtils;
import org.dependencytrack.event.CallbackEvent;
import org.dependencytrack.event.PortfolioMetricsUpdateEvent;
Expand Down Expand Up @@ -68,11 +69,11 @@ private void updateMetrics() throws Exception {
final PersistenceManager pm = qm.getPersistenceManager();

LOGGER.debug("Fetching first " + BATCH_SIZE + " projects");
List<Project> activeProjects = fetchNextActiveProjectsPage(pm, null);
List<Project> activeProjects = fetchNextActiveProjectsBatch(pm, null);

while (!activeProjects.isEmpty()) {
final long firstId = activeProjects.get(0).getId();
final long lastId = activeProjects.get(activeProjects.size() - 1).getId();
final long firstId = activeProjects.getFirst().getId();
final long lastId = activeProjects.getLast().getId();
final int batchCount = activeProjects.size();

final var countDownLatch = new CountDownLatch(batchCount);
Expand Down Expand Up @@ -113,7 +114,7 @@ private void updateMetrics() throws Exception {
counters.medium += metrics.getMedium();
counters.low += metrics.getLow();
counters.unassigned += metrics.getUnassigned();
counters.vulnerabilities += metrics.getVulnerabilities();
counters.vulnerabilities += Math.toIntExact(metrics.getVulnerabilities());

counters.findingsTotal += metrics.getFindingsTotal();
counters.findingsAudited += metrics.getFindingsAudited();
Expand Down Expand Up @@ -145,8 +146,13 @@ private void updateMetrics() throws Exception {
counters.policyViolationsOperationalUnaudited += metrics.getPolicyViolationsOperationalUnaudited();
}

// Remove projects and project metrics from the L1 cache
// to prevent it from growing too large.
pm.evictAll(false, Project.class);
pm.evictAll(false, ProjectMetrics.class);

LOGGER.debug("Fetching next " + BATCH_SIZE + " projects");
activeProjects = fetchNextActiveProjectsPage(pm, lastId);
activeProjects = fetchNextActiveProjectsBatch(pm, lastId);
}

qm.runInTransaction(() -> {
Expand All @@ -166,18 +172,23 @@ private void updateMetrics() throws Exception {
DurationFormatUtils.formatDuration(new Date().getTime() - counters.measuredAt.getTime(), "mm:ss:SS"));
}

private List<Project> fetchNextActiveProjectsPage(final PersistenceManager pm, final Long lastId) throws Exception {
try (final Query<Project> query = pm.newQuery(Project.class)) {
if (lastId == null) {
query.setFilter("(active == null || active == true)");
} else {
query.setFilter("(active == null || active == true) && id < :lastId");
query.setParameters(lastId);
}
query.setOrdering("id DESC");
query.range(0, BATCH_SIZE);
query.getFetchPlan().setGroup(Project.FetchGroup.METRICS_UPDATE.name());
private List<Project> fetchNextActiveProjectsBatch(final PersistenceManager pm, final Long lastId) {
final Query<Project> query = pm.newQuery(Project.class);
if (lastId == null) {
query.setFilter("(active == null || active == true)");
} else {
query.setFilter("(active == null || active == true) && id < :lastId");
query.setParameters(lastId);
}
query.setOrdering("id DESC");
query.range(0, BATCH_SIZE);

// NB: Set fetch group on PM level to avoid fields of the default fetch group from being loaded.
try (var ignoredPersistenceCustomization = new ScopedCustomization(pm)
.withFetchGroup(Project.FetchGroup.METRICS_UPDATE.name())) {
return List.copyOf(query.executeList());
} finally {
query.closeAll();
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import alpine.common.logging.Logger;
import alpine.event.framework.Event;
import alpine.event.framework.Subscriber;
import alpine.persistence.ScopedCustomization;
import org.apache.commons.lang3.time.DurationFormatUtils;
import org.dependencytrack.event.ProjectMetricsUpdateEvent;
import org.dependencytrack.metrics.Metrics;
Expand Down Expand Up @@ -73,6 +74,8 @@ private void updateMetrics(final UUID uuid) throws Exception {
List<Component> components = fetchNextComponentsPage(pm, project, null);

while (!components.isEmpty()) {
final long lastId = components.getLast().getId();

for (final Component component : components) {
final Counters componentCounters;
try {
Expand Down Expand Up @@ -123,8 +126,12 @@ private void updateMetrics(final UUID uuid) throws Exception {
counters.policyViolationsOperationalUnaudited += componentCounters.policyViolationsOperationalUnaudited;
}

// Remove components from the L1 cache to prevent it from growing too large.
// Note that because ComponentMetricsUpdateTask uses its own QueryManager,
// component metrics objects are not in this L1 cache.
pm.evictAll(false, Component.class);

LOGGER.debug("Fetching next components page for project " + uuid);
final long lastId = components.get(components.size() - 1).getId();
components = fetchNextComponentsPage(pm, project, lastId);
}

Expand All @@ -141,7 +148,7 @@ private void updateMetrics(final UUID uuid) throws Exception {
});

if (project.getLastInheritedRiskScore() == null ||
project.getLastInheritedRiskScore() != counters.inheritedRiskScore) {
project.getLastInheritedRiskScore() != counters.inheritedRiskScore) {
LOGGER.debug("Updating inherited risk score of project " + uuid);
qm.runInTransaction(() -> project.setLastInheritedRiskScore(counters.inheritedRiskScore));
}
Expand All @@ -151,19 +158,24 @@ private void updateMetrics(final UUID uuid) throws Exception {
DurationFormatUtils.formatDuration(new Date().getTime() - counters.measuredAt.getTime(), "mm:ss:SS"));
}

private List<Component> fetchNextComponentsPage(final PersistenceManager pm, final Project project, final Long lastId) throws Exception {
try (final Query<Component> query = pm.newQuery(Component.class)) {
if (lastId == null) {
query.setFilter("project == :project");
query.setParameters(project);
} else {
query.setFilter("project == :project && id < :lastId");
query.setParameters(project, lastId);
}
query.setOrdering("id DESC");
query.setRange(0, 500);
query.getFetchPlan().setGroup(Component.FetchGroup.METRICS_UPDATE.name());
private List<Component> fetchNextComponentsPage(final PersistenceManager pm, final Project project, final Long lastId) {
final Query<Component> query = pm.newQuery(Component.class);
if (lastId == null) {
query.setFilter("project == :project");
query.setParameters(project);
} else {
query.setFilter("project == :project && id < :lastId");
query.setParameters(project, lastId);
}
query.setOrdering("id DESC");
query.setRange(0, 1000);

// NB: Set fetch group on PM level to avoid fields of the default fetch group from being loaded.
try (var ignoredPersistenceCustomization = new ScopedCustomization(pm)
.withFetchGroup(Component.FetchGroup.METRICS_UPDATE.name())) {
return List.copyOf(query.executeList());
} finally {
query.closeAll();
}
}

Expand Down

0 comments on commit d2adc7e

Please sign in to comment.