Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Backport: Reduce memory usage of metrics update tasks #4377

Merged
merged 1 commit into from
Nov 14, 2024
Merged

Backport: Reduce memory usage of metrics update tasks #4377

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions src/main/java/org/dependencytrack/tasks/metrics/ComponentMetricsUpdateTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import alpine.common.logging.Logger;
import alpine.event.framework.Event;
import alpine.event.framework.Subscriber;
import alpine.persistence.ScopedCustomization;
import org.apache.commons.lang3.time.DurationFormatUtils;
import org.dependencytrack.event.ComponentMetricsUpdateEvent;
import org.dependencytrack.metrics.Metrics;
Expand Down Expand Up @@ -176,21 +177,26 @@ static Counters updateMetrics(final UUID uuid) throws Exception {
}

@SuppressWarnings("unchecked")
private static List<Vulnerability> getVulnerabilities(final PersistenceManager pm, final Component component) throws Exception {
private static List<Vulnerability> getVulnerabilities(final PersistenceManager pm, final Component component) {
// Using the JDO single-string syntax here because we need to pass the parameter
// of the outer query (the component) to the sub-query. For some reason that does
// not work with the declarative JDO API.
try (final Query<?> query = pm.newQuery(Query.JDOQL, """
final Query<?> query = pm.newQuery(Query.JDOQL, """
SELECT FROM org.dependencytrack.model.Vulnerability
WHERE this.components.contains(:component)
&& (SELECT FROM org.dependencytrack.model.Analysis a
WHERE a.component == :component
&& a.vulnerability == this
&& a.suppressed == true).isEmpty()
""")) {
query.setParameters(component);
query.getFetchPlan().setGroup(Vulnerability.FetchGroup.METRICS_UPDATE.name());
""");
query.setParameters(component);

// NB: Set fetch group on PM level to avoid fields of the default fetch group from being loaded.
try (var ignoredPersistenceCustomization = new ScopedCustomization(pm)
.withFetchGroup(Vulnerability.FetchGroup.METRICS_UPDATE.name())) {
return List.copyOf((List<Vulnerability>) query.executeList());
} finally {
query.closeAll();
}
}

Expand Down
43 changes: 27 additions & 16 deletions src/main/java/org/dependencytrack/tasks/metrics/PortfolioMetricsUpdateTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import alpine.common.util.SystemUtil;
import alpine.event.framework.Event;
import alpine.event.framework.Subscriber;
import alpine.persistence.ScopedCustomization;
import org.apache.commons.lang3.time.DurationFormatUtils;
import org.dependencytrack.event.CallbackEvent;
import org.dependencytrack.event.PortfolioMetricsUpdateEvent;
Expand Down Expand Up @@ -68,11 +69,11 @@ private void updateMetrics() throws Exception {
final PersistenceManager pm = qm.getPersistenceManager();

LOGGER.debug("Fetching first " + BATCH_SIZE + " projects");
List<Project> activeProjects = fetchNextActiveProjectsPage(pm, null);
List<Project> activeProjects = fetchNextActiveProjectsBatch(pm, null);

while (!activeProjects.isEmpty()) {
final long firstId = activeProjects.get(0).getId();
final long lastId = activeProjects.get(activeProjects.size() - 1).getId();
final long firstId = activeProjects.getFirst().getId();
final long lastId = activeProjects.getLast().getId();
final int batchCount = activeProjects.size();

final var countDownLatch = new CountDownLatch(batchCount);
Expand Down Expand Up @@ -113,7 +114,7 @@ private void updateMetrics() throws Exception {
counters.medium += metrics.getMedium();
counters.low += metrics.getLow();
counters.unassigned += metrics.getUnassigned();
counters.vulnerabilities += metrics.getVulnerabilities();
counters.vulnerabilities += Math.toIntExact(metrics.getVulnerabilities());

counters.findingsTotal += metrics.getFindingsTotal();
counters.findingsAudited += metrics.getFindingsAudited();
Expand Down Expand Up @@ -145,8 +146,13 @@ private void updateMetrics() throws Exception {
counters.policyViolationsOperationalUnaudited += metrics.getPolicyViolationsOperationalUnaudited();
}

// Remove projects and project metrics from the L1 cache
// to prevent it from growing too large.
pm.evictAll(false, Project.class);
pm.evictAll(false, ProjectMetrics.class);

LOGGER.debug("Fetching next " + BATCH_SIZE + " projects");
activeProjects = fetchNextActiveProjectsPage(pm, lastId);
activeProjects = fetchNextActiveProjectsBatch(pm, lastId);
}

qm.runInTransaction(() -> {
Expand All @@ -166,18 +172,23 @@ private void updateMetrics() throws Exception {
DurationFormatUtils.formatDuration(new Date().getTime() - counters.measuredAt.getTime(), "mm:ss:SS"));
}

private List<Project> fetchNextActiveProjectsPage(final PersistenceManager pm, final Long lastId) throws Exception {
try (final Query<Project> query = pm.newQuery(Project.class)) {
if (lastId == null) {
query.setFilter("(active == null || active == true)");
} else {
query.setFilter("(active == null || active == true) && id < :lastId");
query.setParameters(lastId);
}
query.setOrdering("id DESC");
query.range(0, BATCH_SIZE);
query.getFetchPlan().setGroup(Project.FetchGroup.METRICS_UPDATE.name());
private List<Project> fetchNextActiveProjectsBatch(final PersistenceManager pm, final Long lastId) {
final Query<Project> query = pm.newQuery(Project.class);
if (lastId == null) {
query.setFilter("(active == null || active == true)");
} else {
query.setFilter("(active == null || active == true) && id < :lastId");
query.setParameters(lastId);
}
query.setOrdering("id DESC");
query.range(0, BATCH_SIZE);

// NB: Set fetch group on PM level to avoid fields of the default fetch group from being loaded.
try (var ignoredPersistenceCustomization = new ScopedCustomization(pm)
.withFetchGroup(Project.FetchGroup.METRICS_UPDATE.name())) {
return List.copyOf(query.executeList());
} finally {
query.closeAll();
}
}

Expand Down
40 changes: 26 additions & 14 deletions src/main/java/org/dependencytrack/tasks/metrics/ProjectMetricsUpdateTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import alpine.common.logging.Logger;
import alpine.event.framework.Event;
import alpine.event.framework.Subscriber;
import alpine.persistence.ScopedCustomization;
import org.apache.commons.lang3.time.DurationFormatUtils;
import org.dependencytrack.event.ProjectMetricsUpdateEvent;
import org.dependencytrack.metrics.Metrics;
Expand Down Expand Up @@ -73,6 +74,8 @@ private void updateMetrics(final UUID uuid) throws Exception {
List<Component> components = fetchNextComponentsPage(pm, project, null);

while (!components.isEmpty()) {
final long lastId = components.getLast().getId();

for (final Component component : components) {
final Counters componentCounters;
try {
Expand Down Expand Up @@ -123,8 +126,12 @@ private void updateMetrics(final UUID uuid) throws Exception {
counters.policyViolationsOperationalUnaudited += componentCounters.policyViolationsOperationalUnaudited;
}

// Remove components from the L1 cache to prevent it from growing too large.
// Note that because ComponentMetricsUpdateTask uses its own QueryManager,
// component metrics objects are not in this L1 cache.
pm.evictAll(false, Component.class);

LOGGER.debug("Fetching next components page for project " + uuid);
final long lastId = components.get(components.size() - 1).getId();
components = fetchNextComponentsPage(pm, project, lastId);
}

Expand All @@ -141,7 +148,7 @@ private void updateMetrics(final UUID uuid) throws Exception {
});

if (project.getLastInheritedRiskScore() == null ||
project.getLastInheritedRiskScore() != counters.inheritedRiskScore) {
project.getLastInheritedRiskScore() != counters.inheritedRiskScore) {
LOGGER.debug("Updating inherited risk score of project " + uuid);
qm.runInTransaction(() -> project.setLastInheritedRiskScore(counters.inheritedRiskScore));
}
Expand All @@ -151,19 +158,24 @@ private void updateMetrics(final UUID uuid) throws Exception {
DurationFormatUtils.formatDuration(new Date().getTime() - counters.measuredAt.getTime(), "mm:ss:SS"));
}

private List<Component> fetchNextComponentsPage(final PersistenceManager pm, final Project project, final Long lastId) throws Exception {
try (final Query<Component> query = pm.newQuery(Component.class)) {
if (lastId == null) {
query.setFilter("project == :project");
query.setParameters(project);
} else {
query.setFilter("project == :project && id < :lastId");
query.setParameters(project, lastId);
}
query.setOrdering("id DESC");
query.setRange(0, 500);
query.getFetchPlan().setGroup(Component.FetchGroup.METRICS_UPDATE.name());
private List<Component> fetchNextComponentsPage(final PersistenceManager pm, final Project project, final Long lastId) {
final Query<Component> query = pm.newQuery(Component.class);
if (lastId == null) {
query.setFilter("project == :project");
query.setParameters(project);
} else {
query.setFilter("project == :project && id < :lastId");
query.setParameters(project, lastId);
}
query.setOrdering("id DESC");
query.setRange(0, 1000);

// NB: Set fetch group on PM level to avoid fields of the default fetch group from being loaded.
try (var ignoredPersistenceCustomization = new ScopedCustomization(pm)
.withFetchGroup(Component.FetchGroup.METRICS_UPDATE.name())) {
return List.copyOf(query.executeList());
} finally {
query.closeAll();
}
}

Expand Down