diff --git a/src/main/java/org/dependencytrack/parser/nvd/NvdParser.java b/src/main/java/org/dependencytrack/parser/nvd/NvdParser.java index 33fbfe855b..1c558dce2c 100644 --- a/src/main/java/org/dependencytrack/parser/nvd/NvdParser.java +++ b/src/main/java/org/dependencytrack/parser/nvd/NvdParser.java @@ -27,13 +27,11 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.commons.lang3.StringUtils; -import org.datanucleus.PropertyNames; import org.dependencytrack.event.IndexEvent; import org.dependencytrack.model.Cwe; import org.dependencytrack.model.Vulnerability; import org.dependencytrack.model.VulnerableSoftware; import org.dependencytrack.parser.common.resolver.CweResolver; -import org.dependencytrack.persistence.QueryManager; import org.dependencytrack.util.VulnerabilityUtil; import us.springett.cvss.Cvss; import us.springett.parsers.cpe.exceptions.CpeEncodingException; @@ -51,6 +49,7 @@ import java.util.Iterator; import java.util.List; import java.util.Optional; +import java.util.function.BiConsumer; /** * Parser and processor of NVD data feeds. @@ -71,6 +70,11 @@ private enum Operator { // https://github.com/DependencyTrack/dependency-track/pull/2520 // is merged. private final ObjectMapper objectMapper = new ObjectMapper(); + private final BiConsumer> vulnerabilityConsumer; + + public NvdParser(final BiConsumer> vulnerabilityConsumer) { + this.vulnerabilityConsumer = vulnerabilityConsumer; + } public void parse(final File file) { if (!file.getName().endsWith(".json")) { @@ -111,127 +115,115 @@ public void parse(final File file) { } private void parseCveItem(final ObjectNode cveItem) { - try (QueryManager qm = new QueryManager().withL2CacheDisabled()) { - qm.getPersistenceManager().setProperty(PropertyNames.PROPERTY_PERSISTENCE_BY_REACHABILITY_AT_COMMIT, "false"); - - final Vulnerability vulnerability = new Vulnerability(); - vulnerability.setSource(Vulnerability.Source.NVD); + final Vulnerability vulnerability = new Vulnerability(); + vulnerability.setSource(Vulnerability.Source.NVD); - // CVE ID - final var cve = (ObjectNode) cveItem.get("cve"); - final var meta0 = (ObjectNode) cve.get("CVE_data_meta"); - vulnerability.setVulnId(meta0.get("ID").asText()); + // CVE ID + final var cve = (ObjectNode) cveItem.get("cve"); + final var meta0 = (ObjectNode) cve.get("CVE_data_meta"); + vulnerability.setVulnId(meta0.get("ID").asText()); - // CVE Published and Modified dates - final String publishedDateString = cveItem.get("publishedDate").asText(); - final String lastModifiedDateString = cveItem.get("lastModifiedDate").asText(); - try { - if (StringUtils.isNotBlank(publishedDateString)) { - vulnerability.setPublished(Date.from(OffsetDateTime.parse(publishedDateString).toInstant())); - } - if (StringUtils.isNotBlank(lastModifiedDateString)) { - vulnerability.setUpdated(Date.from(OffsetDateTime.parse(lastModifiedDateString).toInstant())); - } - } catch (DateTimeParseException | NullPointerException | IllegalArgumentException e) { - LOGGER.error("Unable to parse dates from NVD data feed", e); + // CVE Published and Modified dates + final String publishedDateString = cveItem.get("publishedDate").asText(); + final String lastModifiedDateString = cveItem.get("lastModifiedDate").asText(); + try { + if (StringUtils.isNotBlank(publishedDateString)) { + vulnerability.setPublished(Date.from(OffsetDateTime.parse(publishedDateString).toInstant())); } + if (StringUtils.isNotBlank(lastModifiedDateString)) { + vulnerability.setUpdated(Date.from(OffsetDateTime.parse(lastModifiedDateString).toInstant())); + } + } catch (DateTimeParseException | NullPointerException | IllegalArgumentException e) { + LOGGER.error("Unable to parse dates from NVD data feed", e); + } - // CVE Description - final var descO = (ObjectNode) cve.get("description"); - final var desc1 = (ArrayNode) descO.get("description_data"); - final StringBuilder descriptionBuilder = new StringBuilder(); - for (int j = 0; j < desc1.size(); j++) { - final var desc2 = (ObjectNode) desc1.get(j); - if ("en".equals(desc2.get("lang").asText())) { - descriptionBuilder.append(desc2.get("value").asText()); - if (j < desc1.size() - 1) { - descriptionBuilder.append("\n\n"); - } + // CVE Description + final var descO = (ObjectNode) cve.get("description"); + final var desc1 = (ArrayNode) descO.get("description_data"); + final StringBuilder descriptionBuilder = new StringBuilder(); + for (int j = 0; j < desc1.size(); j++) { + final var desc2 = (ObjectNode) desc1.get(j); + if ("en".equals(desc2.get("lang").asText())) { + descriptionBuilder.append(desc2.get("value").asText()); + if (j < desc1.size() - 1) { + descriptionBuilder.append("\n\n"); } } - vulnerability.setDescription(descriptionBuilder.toString()); + } + vulnerability.setDescription(descriptionBuilder.toString()); - // CVE Impact - parseCveImpact(cveItem, vulnerability); + // CVE Impact + parseCveImpact(cveItem, vulnerability); - // CWE - final var prob0 = (ObjectNode) cve.get("problemtype"); - final var prob1 = (ArrayNode) prob0.get("problemtype_data"); - for (int j = 0; j < prob1.size(); j++) { - final var prob2 = (ObjectNode) prob1.get(j); - final var prob3 = (ArrayNode) prob2.get("description"); - for (int k = 0; k < prob3.size(); k++) { - final var prob4 = (ObjectNode) prob3.get(k); - if ("en".equals(prob4.get("lang").asText())) { - final String cweString = prob4.get("value").asText(); - if (cweString != null && cweString.startsWith("CWE-")) { - final Cwe cwe = CweResolver.getInstance().lookup(cweString); - if (cwe != null) { - vulnerability.addCwe(cwe); - } else { - LOGGER.warn("CWE " + cweString + " not found in Dependency-Track database. This could signify an issue with the NVD or with Dependency-Track not having advanced knowledge of this specific CWE identifier."); - } + // CWE + final var prob0 = (ObjectNode) cve.get("problemtype"); + final var prob1 = (ArrayNode) prob0.get("problemtype_data"); + for (int j = 0; j < prob1.size(); j++) { + final var prob2 = (ObjectNode) prob1.get(j); + final var prob3 = (ArrayNode) prob2.get("description"); + for (int k = 0; k < prob3.size(); k++) { + final var prob4 = (ObjectNode) prob3.get(k); + if ("en".equals(prob4.get("lang").asText())) { + final String cweString = prob4.get("value").asText(); + if (cweString != null && cweString.startsWith("CWE-")) { + final Cwe cwe = CweResolver.getInstance().lookup(cweString); + if (cwe != null) { + vulnerability.addCwe(cwe); + } else { + LOGGER.warn("CWE " + cweString + " not found in Dependency-Track database. This could signify an issue with the NVD or with Dependency-Track not having advanced knowledge of this specific CWE identifier."); } } } } + } - // References - final var ref0 = (ObjectNode) cve.get("references"); - final var ref1 = (ArrayNode) ref0.get("reference_data"); - final StringBuilder sb = new StringBuilder(); - for (int l = 0; l < ref1.size(); l++) { - final var ref2 = (ObjectNode) ref1.get(l); - final Iterator fieldNameIter = ref2.fieldNames(); - while (fieldNameIter.hasNext()) { - final String s = fieldNameIter.next(); - if ("url".equals(s)) { - // Convert reference to Markdown format - final String url = ref2.get("url").asText(); - sb.append("* [").append(url).append("](").append(url).append(")\n"); - } + // References + final var ref0 = (ObjectNode) cve.get("references"); + final var ref1 = (ArrayNode) ref0.get("reference_data"); + final StringBuilder sb = new StringBuilder(); + for (int l = 0; l < ref1.size(); l++) { + final var ref2 = (ObjectNode) ref1.get(l); + final Iterator fieldNameIter = ref2.fieldNames(); + while (fieldNameIter.hasNext()) { + final String s = fieldNameIter.next(); + if ("url".equals(s)) { + // Convert reference to Markdown format + final String url = ref2.get("url").asText(); + sb.append("* [").append(url).append("](").append(url).append(")\n"); } } - final String references = sb.toString(); - if (references.length() > 0) { - vulnerability.setReferences(references.substring(0, references.lastIndexOf("\n"))); - } - - // Update the vulnerability - LOGGER.debug("Synchronizing: " + vulnerability.getVulnId()); - final Vulnerability synchronizeVulnerability = qm.synchronizeVulnerability(vulnerability, false); - final List vsListOld = qm.detach(qm.getVulnerableSoftwareByVulnId(synchronizeVulnerability.getSource(), synchronizeVulnerability.getVulnId())); + } + final String references = sb.toString(); + if (!references.isEmpty()) { + vulnerability.setReferences(references.substring(0, references.lastIndexOf("\n"))); + } - // CPE - List vsList = new ArrayList<>(); - final var configurations = (ObjectNode) cveItem.get("configurations"); - final var nodes = (ArrayNode) configurations.get("nodes"); - for (int j = 0; j < nodes.size(); j++) { - final var node = (ObjectNode) nodes.get(j); - final List vulnerableSoftwareInNode = new ArrayList<>(); - final Operator nodeOperator = Operator.valueOf(node.get("operator").asText(Operator.NONE.name())); - if (node.has("children")) { - // https://github.com/DependencyTrack/dependency-track/issues/1033 - final var children = (ArrayNode) node.get("children"); - if (children.size() > 0) { - for (int l = 0; l < children.size(); l++) { - final var child = (ObjectNode) children.get(l); - vulnerableSoftwareInNode.addAll(parseCpes(qm, child)); - } - } else { - vulnerableSoftwareInNode.addAll(parseCpes(qm, node)); + // CPE + List vsList = new ArrayList<>(); + final var configurations = (ObjectNode) cveItem.get("configurations"); + final var nodes = (ArrayNode) configurations.get("nodes"); + for (int j = 0; j < nodes.size(); j++) { + final var node = (ObjectNode) nodes.get(j); + final List vulnerableSoftwareInNode = new ArrayList<>(); + final Operator nodeOperator = Operator.valueOf(node.get("operator").asText(Operator.NONE.name())); + if (node.has("children")) { + // https://github.com/DependencyTrack/dependency-track/issues/1033 + final var children = (ArrayNode) node.get("children"); + if (!children.isEmpty()) { + for (int l = 0; l < children.size(); l++) { + final var child = (ObjectNode) children.get(l); + vulnerableSoftwareInNode.addAll(parseCpes(child)); } } else { - vulnerableSoftwareInNode.addAll(parseCpes(qm, node)); + vulnerableSoftwareInNode.addAll(parseCpes(node)); } - vsList.addAll(reconcile(vulnerableSoftwareInNode, nodeOperator)); + } else { + vulnerableSoftwareInNode.addAll(parseCpes(node)); } - qm.persist(vsList); - qm.updateAffectedVersionAttributions(synchronizeVulnerability, vsList, Vulnerability.Source.NVD); - vsList = qm.reconcileVulnerableSoftware(synchronizeVulnerability, vsListOld, vsList, Vulnerability.Source.NVD); - synchronizeVulnerability.setVulnerableSoftware(vsList); - qm.persist(synchronizeVulnerability); + vsList.addAll(reconcile(vulnerableSoftwareInNode, nodeOperator)); } + + vulnerabilityConsumer.accept(vulnerability, vsList); } /** @@ -302,14 +294,14 @@ private void parseCveImpact(final ObjectNode cveItem, final Vulnerability vuln) )); } - private List parseCpes(final QueryManager qm, final ObjectNode node) { + private List parseCpes(final ObjectNode node) { final List vsList = new ArrayList<>(); if (node.has("cpe_match")) { final var cpeMatches = (ArrayNode) node.get("cpe_match"); for (int k = 0; k < cpeMatches.size(); k++) { final var cpeMatch = (ObjectNode) cpeMatches.get(k); if (cpeMatch.get("vulnerable").asBoolean(true)) { // only parse the CPEs marked as vulnerable - final VulnerableSoftware vs = generateVulnerableSoftware(qm, cpeMatch); + final VulnerableSoftware vs = generateVulnerableSoftware(cpeMatch); if (vs != null) { vsList.add(vs); } @@ -319,29 +311,26 @@ private List parseCpes(final QueryManager qm, final ObjectNo return vsList; } - private VulnerableSoftware generateVulnerableSoftware(final QueryManager qm, final ObjectNode cpeMatch) { + private VulnerableSoftware generateVulnerableSoftware(final ObjectNode cpeMatch) { final String cpe23Uri = cpeMatch.get("cpe23Uri").asText(); final String versionEndExcluding = Optional.ofNullable(cpeMatch.get("versionEndExcluding")).map(JsonNode::asText).orElse(null); final String versionEndIncluding = Optional.ofNullable(cpeMatch.get("versionEndIncluding")).map(JsonNode::asText).orElse(null); final String versionStartExcluding = Optional.ofNullable(cpeMatch.get("versionStartExcluding")).map(JsonNode::asText).orElse(null); final String versionStartIncluding = Optional.ofNullable(cpeMatch.get("versionStartIncluding")).map(JsonNode::asText).orElse(null); - VulnerableSoftware vs = qm.getVulnerableSoftwareByCpe23(cpe23Uri, versionEndExcluding, - versionEndIncluding, versionStartExcluding, versionStartIncluding); - if (vs != null) { - return vs; - } + + final VulnerableSoftware vs; try { vs = ModelConverter.convertCpe23UriToVulnerableSoftware(cpe23Uri); - vs.setVulnerable(cpeMatch.get("vulnerable").asBoolean(true)); - vs.setVersionEndExcluding(versionEndExcluding); - vs.setVersionEndIncluding(versionEndIncluding); - vs.setVersionStartExcluding(versionStartExcluding); - vs.setVersionStartIncluding(versionStartIncluding); - //Event.dispatch(new IndexEvent(IndexEvent.Action.CREATE, qm.detach(VulnerableSoftware.class, vs.getId()))); - return vs; } catch (CpeParsingException | CpeEncodingException e) { LOGGER.warn("An error occurred while parsing: " + cpe23Uri + " - The CPE is invalid and will be discarded."); + return null; } - return null; + + vs.setVulnerable(cpeMatch.get("vulnerable").asBoolean(true)); + vs.setVersionEndExcluding(versionEndExcluding); + vs.setVersionEndIncluding(versionEndIncluding); + vs.setVersionStartExcluding(versionStartExcluding); + vs.setVersionStartIncluding(versionStartIncluding); + return vs; } } diff --git a/src/main/java/org/dependencytrack/tasks/AbstractNistMirrorTask.java b/src/main/java/org/dependencytrack/tasks/AbstractNistMirrorTask.java new file mode 100644 index 0000000000..5676677fae --- /dev/null +++ b/src/main/java/org/dependencytrack/tasks/AbstractNistMirrorTask.java @@ -0,0 +1,262 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) OWASP Foundation. All Rights Reserved. + */ +package org.dependencytrack.tasks; + +import alpine.common.logging.Logger; +import org.dependencytrack.model.AffectedVersionAttribution; +import org.dependencytrack.model.Vulnerability; +import org.dependencytrack.model.VulnerableSoftware; +import org.dependencytrack.persistence.QueryManager; +import org.dependencytrack.util.PersistenceUtil; + +import javax.jdo.Query; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static java.util.stream.Collectors.groupingBy; +import static org.dependencytrack.util.PersistenceUtil.assertNonPersistentAll; +import static org.dependencytrack.util.PersistenceUtil.assertPersistent; + +/** + * @since 4.11.0 + */ +abstract class AbstractNistMirrorTask { + + private final Logger logger = Logger.getLogger(getClass()); + + Vulnerability synchronizeVulnerability(final QueryManager qm, final Vulnerability vuln) { + PersistenceUtil.assertNonPersistent(vuln, "vuln must not be persistent"); + + return qm.runInTransaction(trx -> { + trx.setSerializeRead(true); // SELECT ... FOR UPDATE + + Vulnerability persistentVuln = getVulnerabilityByCveId(qm, vuln.getVulnId()); + if (persistentVuln == null) { + persistentVuln = qm.getPersistenceManager().makePersistent(vuln); + } else { + final Map diffs = updateVulnerability(persistentVuln, vuln); + if (!diffs.isEmpty()) { + logger.debug("%s has changed: %s".formatted(vuln.getVulnId(), diffs)); + return persistentVuln; + } + + logger.debug("%s has not changed".formatted(vuln.getVulnId())); + } + + return persistentVuln; + }); + } + + void synchronizeVulnerableSoftware(final QueryManager qm, final Vulnerability persistentVuln, final List vsList) { + assertPersistent(persistentVuln, "vuln must be persistent"); + assertNonPersistentAll(vsList, "vsList must not be persistent"); + + qm.runInTransaction(tx -> { + tx.setSerializeRead(false); + + // Get all VulnerableSoftware records that are currently associated with the vulnerability. + // Note: For SOME ODD REASON, duplicate (as in, same database ID and all) VulnerableSoftware + // records are returned, when operating on data that was originally created by the feed-based + // NistMirrorTask. We thus have to deduplicate here. + final List vsOldList = persistentVuln.getVulnerableSoftware().stream().distinct().toList(); + logger.trace("%s: Existing VS: %d".formatted(persistentVuln.getVulnId(), vsOldList.size())); + + // Get attributions for all existing VulnerableSoftware records. + final Map> attributionsByVsId = + qm.getAffectedVersionAttributions(persistentVuln, vsOldList).stream() + .collect(groupingBy(attribution -> attribution.getVulnerableSoftware().getId())); + for (final VulnerableSoftware vsOld : vsOldList) { + vsOld.setAffectedVersionAttributions(attributionsByVsId.get(vsOld.getId())); + } + + // Based on the lists of currently reported, and previously reported VulnerableSoftware records, + // divide the previously reported ones into lists of records to keep, and records to remove. + // Records to keep are removed from vsList. Remaining records in vsList thus are entirely new. + final var vsListToRemove = new ArrayList(); + final var vsListToKeep = new ArrayList(); + for (final VulnerableSoftware vsOld : vsOldList) { + if (vsList.removeIf(vsOld::equalsIgnoringDatastoreIdentity)) { + vsListToKeep.add(vsOld); + } else { + final List attributions = vsOld.getAffectedVersionAttributions(); + if (attributions == null || attributions.isEmpty()) { + // DT versions prior to 4.7.0 did not record attributions. + // Drop the VulnerableSoftware for now. If it was previously + // reported by another source, it will be recorded and attributed + // whenever that source is mirrored again. + vsListToRemove.add(vsOld); + continue; + } + + final boolean previouslyReportedByNvd = attributions.stream() + .anyMatch(attr -> attr.getSource() == Vulnerability.Source.NVD); + final boolean previouslyReportedByOthers = !previouslyReportedByNvd; + + if (previouslyReportedByOthers) { + vsListToKeep.add(vsOld); + } else { + vsListToRemove.add(vsOld); + } + } + } + logger.trace("%s: vsListToKeep: %d".formatted(persistentVuln.getVulnId(), vsListToKeep.size())); + logger.trace("%s: vsListToRemove: %d".formatted(persistentVuln.getVulnId(), vsListToRemove.size())); + + // Remove attributions for VulnerableSoftware records that are no longer reported. + if (!vsListToRemove.isEmpty()) { + qm.deleteAffectedVersionAttributions(persistentVuln, vsListToRemove, Vulnerability.Source.NVD); + } + + final var attributionDate = new Date(); + + // For VulnerableSoftware records that existed before, update the lastSeen timestamp. + for (final VulnerableSoftware oldVs : vsListToKeep) { + oldVs.getAffectedVersionAttributions().stream() + .filter(attribution -> attribution.getSource() == Vulnerability.Source.NVD) + .findAny() + .ifPresent(attribution -> attribution.setLastSeen(attributionDate)); + } + + // For VulnerableSoftware records that are newly reported for this vulnerability, check if any matching + // records exist in the database that are currently associated with other (or no) vulnerabilities. + for (final VulnerableSoftware vs : vsList) { + final VulnerableSoftware existingVs = qm.getVulnerableSoftwareByCpe23( + vs.getCpe23(), + vs.getVersionEndExcluding(), + vs.getVersionEndIncluding(), + vs.getVersionStartExcluding(), + vs.getVersionStartIncluding() + ); + if (existingVs != null) { + final boolean hasAttribution = qm.hasAffectedVersionAttribution(persistentVuln, existingVs, Vulnerability.Source.NVD); + if (!hasAttribution) { + logger.trace("%s: Adding attribution".formatted(persistentVuln.getVulnId())); + final AffectedVersionAttribution attribution = createAttribution(persistentVuln, existingVs, attributionDate); + qm.getPersistenceManager().makePersistent(attribution); + } else { + logger.debug("%s: Encountered dangling attribution; Re-using by updating firstSeen and lastSeen timestamps".formatted(persistentVuln.getVulnId())); + final AffectedVersionAttribution existingAttribution = qm.getAffectedVersionAttribution(persistentVuln, existingVs, Vulnerability.Source.NVD); + existingAttribution.setFirstSeen(attributionDate); + existingAttribution.setLastSeen(attributionDate); + } + vsListToKeep.add(existingVs); + } else { + logger.trace("%s: Creating new VS".formatted(persistentVuln.getVulnId())); + final VulnerableSoftware persistentVs = qm.getPersistenceManager().makePersistent(vs); + final AffectedVersionAttribution attribution = createAttribution(persistentVuln, persistentVs, attributionDate); + qm.getPersistenceManager().makePersistent(attribution); + vsListToKeep.add(persistentVs); + } + } + + logger.trace("%s: Final vsList: %d".formatted(persistentVuln.getVulnId(), vsListToKeep.size())); + if (!Objects.equals(persistentVuln.getVulnerableSoftware(), vsListToKeep)) { + logger.trace("%s: vsList has changed: %s".formatted(persistentVuln.getVulnId(), new PersistenceUtil.Diff(persistentVuln.getVulnerableSoftware(), vsListToKeep))); + persistentVuln.setVulnerableSoftware(vsListToKeep); + } + }); + } + + private static AffectedVersionAttribution createAttribution(final Vulnerability vuln, final VulnerableSoftware vs, + final Date attributionDate) { + final var attribution = new AffectedVersionAttribution(); + attribution.setSource(Vulnerability.Source.NVD); + attribution.setVulnerability(vuln); + attribution.setVulnerableSoftware(vs); + attribution.setFirstSeen(attributionDate); + attribution.setLastSeen(attributionDate); + return attribution; + } + + /** + * Get a {@link Vulnerability} by its CVE ID (implying the source {@link Vulnerability.Source#NVD}). + *

+ * It differs from {@link QueryManager#getVulnerabilityByVulnId(String, String)} in that it does not fetch any + * adjacent relationships (e.g. affected components and aliases). + * + * @param qm The {@link QueryManager} to use + * @param cveId The CVE ID to look for + * @return The {@link Vulnerability} matching the CVE ID, or {@code null} when no match was found + */ + private static Vulnerability getVulnerabilityByCveId(final QueryManager qm, final String cveId) { + final Query query = qm.getPersistenceManager().newQuery(Vulnerability.class); + query.setFilter("source == :source && vulnId == :cveId"); + query.setNamedParameters(Map.of( + "source", Vulnerability.Source.NVD.name(), + "cveId", cveId + )); + try { + return query.executeUnique(); + } finally { + query.closeAll(); + } + } + + /** + * Update an existing, persistent {@link Vulnerability} with data as reported by the NVD. + *

+ * It differs from {@link QueryManager#updateVulnerability(Vulnerability, boolean)} in that it keeps track of + * which fields are modified, and assumes the to-be-updated {@link Vulnerability} to be persistent, and enrolled + * in an active {@link javax.jdo.Transaction}. + * + * @param existingVuln The existing {@link Vulnerability} to update + * @param reportedVuln The {@link Vulnerability} as reported by the NVD + * @return A {@link Map} holding the differences of all updated fields + */ + private static Map updateVulnerability(final Vulnerability existingVuln, final Vulnerability reportedVuln) { + assertPersistent(existingVuln, "existingVuln must be persistent in order for changes to be effective"); + + final var differ = new PersistenceUtil.Differ<>(existingVuln, reportedVuln); + differ.applyIfChanged("title", Vulnerability::getTitle, existingVuln::setTitle); + differ.applyIfChanged("subTitle", Vulnerability::getSubTitle, existingVuln::setSubTitle); + differ.applyIfChanged("description", Vulnerability::getDescription, existingVuln::setDescription); + differ.applyIfChanged("detail", Vulnerability::getDetail, existingVuln::setDetail); + differ.applyIfChanged("recommendation", Vulnerability::getRecommendation, existingVuln::setRecommendation); + differ.applyIfChanged("references", Vulnerability::getReferences, existingVuln::setReferences); + differ.applyIfChanged("credits", Vulnerability::getCredits, existingVuln::setCredits); + differ.applyIfChanged("created", Vulnerability::getCreated, existingVuln::setCreated); + differ.applyIfChanged("published", Vulnerability::getPublished, existingVuln::setPublished); + differ.applyIfChanged("updated", Vulnerability::getUpdated, existingVuln::setUpdated); + differ.applyIfNonEmptyAndChanged("cwes", Vulnerability::getCwes, existingVuln::setCwes); + differ.applyIfChanged("severity", Vulnerability::getSeverity, existingVuln::setSeverity); + differ.applyIfChanged("cvssV2BaseScore", Vulnerability::getCvssV2BaseScore, existingVuln::setCvssV2BaseScore); + differ.applyIfChanged("cvssV2ImpactSubScore", Vulnerability::getCvssV2ImpactSubScore, existingVuln::setCvssV2ImpactSubScore); + differ.applyIfChanged("cvssV2ExploitabilitySubScore", Vulnerability::getCvssV2ExploitabilitySubScore, existingVuln::setCvssV2ExploitabilitySubScore); + differ.applyIfChanged("cvssV2Vector", Vulnerability::getCvssV2Vector, existingVuln::setCvssV2Vector); + differ.applyIfChanged("cvssV3BaseScore", Vulnerability::getCvssV3BaseScore, existingVuln::setCvssV3BaseScore); + differ.applyIfChanged("cvssV3ImpactSubScore", Vulnerability::getCvssV3ImpactSubScore, existingVuln::setCvssV3ImpactSubScore); + differ.applyIfChanged("cvssV3ExploitabilitySubScore", Vulnerability::getCvssV3ExploitabilitySubScore, existingVuln::setCvssV3ExploitabilitySubScore); + differ.applyIfChanged("cvssV3Vector", Vulnerability::getCvssV3Vector, existingVuln::setCvssV3Vector); + differ.applyIfChanged("owaspRRLikelihoodScore", Vulnerability::getOwaspRRLikelihoodScore, existingVuln::setOwaspRRLikelihoodScore); + differ.applyIfChanged("owaspRRTechnicalImpactScore", Vulnerability::getOwaspRRTechnicalImpactScore, existingVuln::setOwaspRRTechnicalImpactScore); + differ.applyIfChanged("owaspRRBusinessImpactScore", Vulnerability::getOwaspRRBusinessImpactScore, existingVuln::setOwaspRRBusinessImpactScore); + differ.applyIfChanged("owaspRRVector", Vulnerability::getOwaspRRVector, existingVuln::setOwaspRRVector); + differ.applyIfChanged("vulnerableVersions", Vulnerability::getVulnerableVersions, existingVuln::setVulnerableVersions); + differ.applyIfChanged("patchedVersions", Vulnerability::getPatchedVersions, existingVuln::setPatchedVersions); + // EPSS is an additional enrichment that no source currently provides natively. We don't want EPSS scores of CVEs to be purged. + differ.applyIfNonNullAndChanged("epssScore", Vulnerability::getEpssScore, existingVuln::setEpssScore); + differ.applyIfNonNullAndChanged("epssPercentile", Vulnerability::getEpssPercentile, existingVuln::setEpssPercentile); + + return differ.getDiffs(); + } + +} diff --git a/src/main/java/org/dependencytrack/tasks/NistApiMirrorTask.java b/src/main/java/org/dependencytrack/tasks/NistApiMirrorTask.java index 6461c6b8ac..d82ac2ae8e 100644 --- a/src/main/java/org/dependencytrack/tasks/NistApiMirrorTask.java +++ b/src/main/java/org/dependencytrack/tasks/NistApiMirrorTask.java @@ -33,7 +33,6 @@ import io.github.jeremylong.openvulnerability.client.nvd.NvdCveClientBuilder; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.concurrent.BasicThreadFactory; -import org.apache.commons.lang3.tuple.Pair; import org.apache.hc.client5.http.auth.AuthScope; import org.apache.hc.client5.http.auth.NTCredentials; import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; @@ -45,27 +44,20 @@ import org.dependencytrack.common.AlpineHttpProxySelector; import org.dependencytrack.event.EpssMirrorEvent; import org.dependencytrack.event.IndexEvent; -import org.dependencytrack.event.IndexEvent.Action; import org.dependencytrack.event.NistApiMirrorEvent; import org.dependencytrack.model.AffectedVersionAttribution; import org.dependencytrack.model.Vulnerability; -import org.dependencytrack.model.Vulnerability.Source; import org.dependencytrack.model.VulnerableSoftware; import org.dependencytrack.persistence.QueryManager; +import org.dependencytrack.persistence.listener.IndexingInstanceLifecycleListener; +import org.dependencytrack.persistence.listener.L2CacheEvictingInstanceLifecycleListener; import org.dependencytrack.util.DebugDataEncryption; -import org.dependencytrack.util.PersistenceUtil.Diff; -import org.dependencytrack.util.PersistenceUtil.Differ; -import javax.jdo.Query; import java.time.Duration; import java.time.Instant; import java.time.ZoneOffset; import java.time.ZonedDateTime; -import java.util.ArrayList; -import java.util.Date; import java.util.List; -import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; @@ -73,22 +65,21 @@ import java.util.concurrent.atomic.AtomicInteger; import static io.github.jeremylong.openvulnerability.client.nvd.NvdCveClientBuilder.aNvdCveApi; -import static java.util.stream.Collectors.groupingBy; import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.datanucleus.PropertyNames.PROPERTY_PERSISTENCE_BY_REACHABILITY_AT_COMMIT; +import static org.datanucleus.PropertyNames.PROPERTY_RETAIN_VALUES; import static org.dependencytrack.model.ConfigPropertyConstants.VULNERABILITY_SOURCE_NVD_API_KEY; import static org.dependencytrack.model.ConfigPropertyConstants.VULNERABILITY_SOURCE_NVD_API_LAST_MODIFIED_EPOCH_SECONDS; import static org.dependencytrack.model.ConfigPropertyConstants.VULNERABILITY_SOURCE_NVD_API_URL; import static org.dependencytrack.parser.nvd.api20.ModelConverter.convert; import static org.dependencytrack.parser.nvd.api20.ModelConverter.convertConfigurations; -import static org.dependencytrack.util.PersistenceUtil.assertPersistent; /** * A {@link Subscriber} that mirrors the content of the NVD through the NVD API 2.0. * * @since 4.10.0 */ -public class NistApiMirrorTask implements Subscriber { +public class NistApiMirrorTask extends AbstractNistMirrorTask implements Subscriber { private static final Logger LOGGER = Logger.getLogger(NistApiMirrorTask.class); @@ -175,12 +166,15 @@ public void inform(final Event e) { executor.submit(() -> { try (final var qm = new QueryManager().withL2CacheDisabled()) { qm.getPersistenceManager().setProperty(PROPERTY_PERSISTENCE_BY_REACHABILITY_AT_COMMIT, "false"); + qm.getPersistenceManager().setProperty(PROPERTY_RETAIN_VALUES, "true"); + qm.getPersistenceManager().addInstanceLifecycleListener(new IndexingInstanceLifecycleListener(Event::dispatch), + Vulnerability.class, VulnerableSoftware.class); + qm.getPersistenceManager().addInstanceLifecycleListener(new L2CacheEvictingInstanceLifecycleListener(qm), + AffectedVersionAttribution.class, Vulnerability.class, VulnerableSoftware.class); - // Note: persistentVuln is in HOLLOW state (all fields except ID are unloaded). - // https://www.datanucleus.org/products/accessplatform_6_0/jdo/persistence.html#lifecycle final Vulnerability persistentVuln = synchronizeVulnerability(qm, vuln); synchronizeVulnerableSoftware(qm, persistentVuln, vsList); - } catch (Exception ex) { + } catch (RuntimeException ex) { LOGGER.error("An unexpected error occurred while processing %s".formatted(vuln.getVulnId()), ex); } finally { final int currentNumMirrored = numMirrored.incrementAndGet(); @@ -225,149 +219,13 @@ public void inform(final Event e) { } if (updateLastModified(lastModified)) { - Event.dispatch(new IndexEvent(Action.COMMIT, Vulnerability.class)); + Event.dispatch(new IndexEvent(IndexEvent.Action.COMMIT, Vulnerability.class)); + Event.dispatch(new IndexEvent(IndexEvent.Action.COMMIT, VulnerableSoftware.class)); } Event.dispatch(new EpssMirrorEvent()); } - private static Vulnerability synchronizeVulnerability(final QueryManager qm, final Vulnerability vuln) { - final Pair vulnIndexEventPair = qm.runInTransaction(trx -> { - trx.setSerializeRead(true); // SELECT ... FOR UPDATE - - Vulnerability persistentVuln = getVulnerabilityByCveId(qm, vuln.getVulnId()); - if (persistentVuln == null) { - persistentVuln = qm.getPersistenceManager().makePersistent(vuln); - return Pair.of(persistentVuln, new IndexEvent(Action.CREATE, persistentVuln)); - } else { - final Map diffs = updateVulnerability(persistentVuln, vuln); - if (!diffs.isEmpty()) { - LOGGER.debug("%s has changed: %s".formatted(vuln.getVulnId(), diffs)); - return Pair.of(persistentVuln, new IndexEvent(Action.UPDATE, persistentVuln)); - } - - LOGGER.debug("%s has not changed".formatted(vuln.getVulnId())); - return Pair.of(persistentVuln, null); - } - }); - - final IndexEvent indexEvent = vulnIndexEventPair.getRight(); - final Vulnerability persistentVuln = vulnIndexEventPair.getLeft(); - - if (indexEvent != null) { - Event.dispatch(indexEvent); - } - - return persistentVuln; - } - - private static void synchronizeVulnerableSoftware(final QueryManager qm, final Vulnerability persistentVuln, final List vsList) { - qm.runInTransaction(tx -> { - tx.setSerializeRead(false); - - // Get all VulnerableSoftware records that are currently associated with the vulnerability. - // Note: For SOME ODD REASON, duplicate (as in, same database ID and all) VulnerableSoftware - // records are returned, when operating on data that was originally created by the feed-based - // NistMirrorTask. We thus have to deduplicate here. - final List vsOldList = persistentVuln.getVulnerableSoftware().stream().distinct().toList(); - LOGGER.trace("%s: Existing VS: %d".formatted(persistentVuln.getVulnId(), vsOldList.size())); - - // Get attributions for all existing VulnerableSoftware records. - final Map> attributionsByVsId = - qm.getAffectedVersionAttributions(persistentVuln, vsOldList).stream() - .collect(groupingBy(attribution -> attribution.getVulnerableSoftware().getId())); - for (final VulnerableSoftware vsOld : vsOldList) { - vsOld.setAffectedVersionAttributions(attributionsByVsId.get(vsOld.getId())); - } - - // Based on the lists of currently reported, and previously reported VulnerableSoftware records, - // divide the previously reported ones into lists of records to keep, and records to remove. - // Records to keep are removed from vsList. Remaining records in vsList thus are entirely new. - final var vsListToRemove = new ArrayList(); - final var vsListToKeep = new ArrayList(); - for (final VulnerableSoftware vsOld : vsOldList) { - if (vsList.removeIf(vsOld::equalsIgnoringDatastoreIdentity)) { - vsListToKeep.add(vsOld); - } else { - final List attributions = vsOld.getAffectedVersionAttributions(); - if (attributions == null || attributions.isEmpty()) { - // DT versions prior to 4.7.0 did not record attributions. - // Drop the VulnerableSoftware for now. If it was previously - // reported by another source, it will be recorded and attributed - // whenever that source is mirrored again. - vsListToRemove.add(vsOld); - continue; - } - - final boolean previouslyReportedByNvd = attributions.stream() - .anyMatch(attr -> attr.getSource() == Source.NVD); - final boolean previouslyReportedByOthers = !previouslyReportedByNvd; - - if (previouslyReportedByOthers) { - vsListToKeep.add(vsOld); - } else { - vsListToRemove.add(vsOld); - } - } - } - LOGGER.trace("%s: vsListToKeep: %d".formatted(persistentVuln.getVulnId(), vsListToKeep.size())); - LOGGER.trace("%s: vsListToRemove: %d".formatted(persistentVuln.getVulnId(), vsListToRemove.size())); - - // Remove attributions for VulnerableSoftware records that are no longer reported. - if (!vsListToRemove.isEmpty()) { - qm.deleteAffectedVersionAttributions(persistentVuln, vsListToRemove, Source.NVD); - } - - final var attributionDate = new Date(); - - // For VulnerableSoftware records that existed before, update the lastSeen timestamp. - for (final VulnerableSoftware oldVs : vsListToKeep) { - oldVs.getAffectedVersionAttributions().stream() - .filter(attribution -> attribution.getSource() == Source.NVD) - .findAny() - .ifPresent(attribution -> attribution.setLastSeen(attributionDate)); - } - - // For VulnerableSoftware records that are newly reported for this vulnerability, check if any matching - // records exist in the database that are currently associated with other (or no) vulnerabilities. - for (final VulnerableSoftware vs : vsList) { - final VulnerableSoftware existingVs = qm.getVulnerableSoftwareByCpe23( - vs.getCpe23(), - vs.getVersionEndExcluding(), - vs.getVersionEndIncluding(), - vs.getVersionStartExcluding(), - vs.getVersionStartIncluding() - ); - if (existingVs != null) { - final boolean hasAttribution = qm.hasAffectedVersionAttribution(persistentVuln, existingVs, Source.NVD); - if (!hasAttribution) { - LOGGER.trace("%s: Adding attribution".formatted(persistentVuln.getVulnId())); - final AffectedVersionAttribution attribution = createAttribution(persistentVuln, existingVs, attributionDate); - qm.getPersistenceManager().makePersistent(attribution); - } else { - LOGGER.debug("%s: Encountered dangling attribution; Re-using by updating firstSeen and lastSeen timestamps".formatted(persistentVuln.getVulnId())); - final AffectedVersionAttribution existingAttribution = qm.getAffectedVersionAttribution(persistentVuln, existingVs, Source.NVD); - existingAttribution.setFirstSeen(attributionDate); - existingAttribution.setLastSeen(attributionDate); - } - vsListToKeep.add(existingVs); - } else { - LOGGER.trace("%s: Creating new VS".formatted(persistentVuln.getVulnId())); - final VulnerableSoftware persistentVs = qm.getPersistenceManager().makePersistent(vs); - final AffectedVersionAttribution attribution = createAttribution(persistentVuln, persistentVs, attributionDate); - qm.getPersistenceManager().makePersistent(attribution); - vsListToKeep.add(persistentVs); - } - } - - LOGGER.trace("%s: Final vsList: %d".formatted(persistentVuln.getVulnId(), vsListToKeep.size())); - if (!Objects.equals(persistentVuln.getVulnerableSoftware(), vsListToKeep)) { - LOGGER.trace("%s: vsList has changed: %s".formatted(persistentVuln.getVulnId(), new Diff(persistentVuln.getVulnerableSoftware(), vsListToKeep))); - persistentVuln.setVulnerableSoftware(vsListToKeep); - } - }); - } - private static NvdCveClient createApiClient(final String apiUrl, final String apiKey, final long lastModifiedEpochSeconds) { final NvdCveClientBuilder clientBuilder = aNvdCveApi() .withHttpClientSupplier(new HttpClientSupplier(apiKey != null)) @@ -409,92 +267,6 @@ private static boolean updateLastModified(final ZonedDateTime lastModifiedDateTi return true; } - private static AffectedVersionAttribution createAttribution(final Vulnerability vuln, final VulnerableSoftware vs, - final Date attributionDate) { - final var attribution = new AffectedVersionAttribution(); - attribution.setSource(Source.NVD); - attribution.setVulnerability(vuln); - attribution.setVulnerableSoftware(vs); - attribution.setFirstSeen(attributionDate); - attribution.setLastSeen(attributionDate); - return attribution; - } - - /** - * Get a {@link Vulnerability} by its CVE ID (implying the source {@link Source#NVD}). - *

- * It differs from {@link QueryManager#getVulnerabilityByVulnId(String, String)} in that it does not fetch any - * adjacent relationships (e.g. affected components and aliases). - * - * @param qm The {@link QueryManager} to use - * @param cveId The CVE ID to look for - * @return The {@link Vulnerability} matching the CVE ID, or {@code null} when no match was found - */ - private static Vulnerability getVulnerabilityByCveId(final QueryManager qm, final String cveId) { - final Query query = qm.getPersistenceManager().newQuery(Vulnerability.class); - query.setFilter("source == :source && vulnId == :cveId"); - query.setNamedParameters(Map.of( - "source", Source.NVD.name(), - "cveId", cveId - )); - try { - return query.executeUnique(); - } finally { - query.closeAll(); - } - } - - /** - * Update an existing, persistent {@link Vulnerability} with data as reported by the NVD. - *

- * It differs from {@link QueryManager#updateVulnerability(Vulnerability, boolean)} in that it keeps track of - * which fields are modified, and assumes the to-be-updated {@link Vulnerability} to be persistent, and enrolled - * in an active {@link javax.jdo.Transaction}. - * - * @param existingVuln The existing {@link Vulnerability} to update - * @param reportedVuln The {@link Vulnerability} as reported by the NVD - * @return A {@link Map} holding the differences of all updated fields - */ - private static Map updateVulnerability(final Vulnerability existingVuln, final Vulnerability reportedVuln) { - assertPersistent(existingVuln, "existingVuln must be persistent in order for changes to be effective"); - - final var differ = new Differ<>(existingVuln, reportedVuln); - differ.applyIfChanged("title", Vulnerability::getTitle, existingVuln::setTitle); - differ.applyIfChanged("subTitle", Vulnerability::getSubTitle, existingVuln::setSubTitle); - differ.applyIfChanged("description", Vulnerability::getDescription, existingVuln::setDescription); - differ.applyIfChanged("detail", Vulnerability::getDetail, existingVuln::setDetail); - differ.applyIfChanged("recommendation", Vulnerability::getRecommendation, existingVuln::setRecommendation); - differ.applyIfChanged("references", Vulnerability::getReferences, existingVuln::setReferences); - differ.applyIfChanged("credits", Vulnerability::getCredits, existingVuln::setCredits); - differ.applyIfChanged("created", Vulnerability::getCreated, existingVuln::setCreated); - differ.applyIfChanged("published", Vulnerability::getPublished, existingVuln::setPublished); - differ.applyIfChanged("updated", Vulnerability::getUpdated, existingVuln::setUpdated); - differ.applyIfNonEmptyAndChanged("cwes", Vulnerability::getCwes, existingVuln::setCwes); - // Calling setSeverity nulls all CVSS and OWASP RR fields. getSeverity calculates the severity on-the-fly, - // and will return UNASSIGNED even when no severity is set explicitly. Thus, calling setSeverity - // must happen before CVSS and OWASP RR fields are set, to avoid null-ing them again. - differ.applyIfChanged("severity", Vulnerability::getSeverity, existingVuln::setSeverity); - differ.applyIfChanged("cvssV2BaseScore", Vulnerability::getCvssV2BaseScore, existingVuln::setCvssV2BaseScore); - differ.applyIfChanged("cvssV2ImpactSubScore", Vulnerability::getCvssV2ImpactSubScore, existingVuln::setCvssV2ImpactSubScore); - differ.applyIfChanged("cvssV2ExploitabilitySubScore", Vulnerability::getCvssV2ExploitabilitySubScore, existingVuln::setCvssV2ExploitabilitySubScore); - differ.applyIfChanged("cvssV2Vector", Vulnerability::getCvssV2Vector, existingVuln::setCvssV2Vector); - differ.applyIfChanged("cvssV3BaseScore", Vulnerability::getCvssV3BaseScore, existingVuln::setCvssV3BaseScore); - differ.applyIfChanged("cvssV3ImpactSubScore", Vulnerability::getCvssV3ImpactSubScore, existingVuln::setCvssV3ImpactSubScore); - differ.applyIfChanged("cvssV3ExploitabilitySubScore", Vulnerability::getCvssV3ExploitabilitySubScore, existingVuln::setCvssV3ExploitabilitySubScore); - differ.applyIfChanged("cvssV3Vector", Vulnerability::getCvssV3Vector, existingVuln::setCvssV3Vector); - differ.applyIfChanged("owaspRRLikelihoodScore", Vulnerability::getOwaspRRLikelihoodScore, existingVuln::setOwaspRRLikelihoodScore); - differ.applyIfChanged("owaspRRTechnicalImpactScore", Vulnerability::getOwaspRRTechnicalImpactScore, existingVuln::setOwaspRRTechnicalImpactScore); - differ.applyIfChanged("owaspRRBusinessImpactScore", Vulnerability::getOwaspRRBusinessImpactScore, existingVuln::setOwaspRRBusinessImpactScore); - differ.applyIfChanged("owaspRRVector", Vulnerability::getOwaspRRVector, existingVuln::setOwaspRRVector); - differ.applyIfChanged("vulnerableVersions", Vulnerability::getVulnerableVersions, existingVuln::setVulnerableVersions); - differ.applyIfChanged("patchedVersions", Vulnerability::getPatchedVersions, existingVuln::setPatchedVersions); - // EPSS is an additional enrichment that no source currently provides natively. We don't want EPSS scores of CVEs to be purged. - differ.applyIfNonNullAndChanged("epssScore", Vulnerability::getEpssScore, existingVuln::setEpssScore); - differ.applyIfNonNullAndChanged("epssPercentile", Vulnerability::getEpssPercentile, existingVuln::setEpssPercentile); - - return differ.getDiffs(); - } - private static final class HttpClientSupplier implements HttpAsyncClientSupplier { private final boolean isApiKeyProvided; diff --git a/src/main/java/org/dependencytrack/tasks/NistMirrorTask.java b/src/main/java/org/dependencytrack/tasks/NistMirrorTask.java index 85d1ab6d04..58901c4e94 100644 --- a/src/main/java/org/dependencytrack/tasks/NistMirrorTask.java +++ b/src/main/java/org/dependencytrack/tasks/NistMirrorTask.java @@ -41,13 +41,19 @@ import org.apache.http.conn.ConnectTimeoutException; import org.dependencytrack.common.HttpClientPool; import org.dependencytrack.event.EpssMirrorEvent; +import org.dependencytrack.event.IndexEvent; import org.dependencytrack.event.NistApiMirrorEvent; import org.dependencytrack.event.NistMirrorEvent; +import org.dependencytrack.model.AffectedVersionAttribution; +import org.dependencytrack.model.Vulnerability; +import org.dependencytrack.model.VulnerableSoftware; import org.dependencytrack.notification.NotificationConstants; import org.dependencytrack.notification.NotificationGroup; import org.dependencytrack.notification.NotificationScope; import org.dependencytrack.parser.nvd.NvdParser; import org.dependencytrack.persistence.QueryManager; +import org.dependencytrack.persistence.listener.IndexingInstanceLifecycleListener; +import org.dependencytrack.persistence.listener.L2CacheEvictingInstanceLifecycleListener; import java.io.BufferedReader; import java.io.Closeable; @@ -64,9 +70,12 @@ import java.time.Duration; import java.util.Calendar; import java.util.Date; +import java.util.List; import java.util.zip.GZIPInputStream; import static io.github.resilience4j.core.IntervalFunction.ofExponentialBackoff; +import static org.datanucleus.PropertyNames.PROPERTY_PERSISTENCE_BY_REACHABILITY_AT_COMMIT; +import static org.datanucleus.PropertyNames.PROPERTY_RETAIN_VALUES; import static org.dependencytrack.model.ConfigPropertyConstants.VULNERABILITY_SOURCE_NVD_API_DOWNLOAD_FEEDS; import static org.dependencytrack.model.ConfigPropertyConstants.VULNERABILITY_SOURCE_NVD_API_ENABLED; import static org.dependencytrack.model.ConfigPropertyConstants.VULNERABILITY_SOURCE_NVD_ENABLED; @@ -78,7 +87,7 @@ * @author Steve Springett * @since 3.0.0 */ -public class NistMirrorTask implements LoggableSubscriber { +public class NistMirrorTask extends AbstractNistMirrorTask implements LoggableSubscriber { private enum ResourceType { CVE_YEAR_DATA, @@ -363,8 +372,10 @@ private void uncompress(final File file, final ResourceType resourceType) { final long start = System.currentTimeMillis(); if (ResourceType.CVE_YEAR_DATA == resourceType || ResourceType.CVE_MODIFIED_DATA == resourceType) { if (!isApiEnabled) { - final NvdParser parser = new NvdParser(); + final NvdParser parser = new NvdParser(this::processVulnerability); parser.parse(uncompressedFile); + Event.dispatch(new IndexEvent(IndexEvent.Action.COMMIT, Vulnerability.class)); + Event.dispatch(new IndexEvent(IndexEvent.Action.COMMIT, VulnerableSoftware.class)); } else { LOGGER.debug(""" %s was successfully downloaded and uncompressed, but will not be parsed because \ @@ -378,6 +389,22 @@ private void uncompress(final File file, final ResourceType resourceType) { metricParseTime += end - start; } + private void processVulnerability(final Vulnerability vuln, final List vsList) { + try (final var qm = new QueryManager().withL2CacheDisabled()) { + qm.getPersistenceManager().setProperty(PROPERTY_PERSISTENCE_BY_REACHABILITY_AT_COMMIT, "false"); + qm.getPersistenceManager().setProperty(PROPERTY_RETAIN_VALUES, "true"); + qm.getPersistenceManager().addInstanceLifecycleListener(new IndexingInstanceLifecycleListener(Event::dispatch), + Vulnerability.class, VulnerableSoftware.class); + qm.getPersistenceManager().addInstanceLifecycleListener(new L2CacheEvictingInstanceLifecycleListener(qm), + AffectedVersionAttribution.class, Vulnerability.class, VulnerableSoftware.class); + + final Vulnerability persistentVuln = synchronizeVulnerability(qm, vuln); + synchronizeVulnerableSoftware(qm, persistentVuln, vsList); + } catch (RuntimeException e) { + LOGGER.error("An unexpected error occurred while processing %s".formatted(vuln.getVulnId()), e); + } + } + /** * Closes a closable object. * @param object the object to close diff --git a/src/main/java/org/dependencytrack/util/PersistenceUtil.java b/src/main/java/org/dependencytrack/util/PersistenceUtil.java index a2c0a73da2..f92ac11abd 100644 --- a/src/main/java/org/dependencytrack/util/PersistenceUtil.java +++ b/src/main/java/org/dependencytrack/util/PersistenceUtil.java @@ -201,6 +201,22 @@ public static void assertNonPersistent(final Object object, final String message } } + /** + * Utility method to ensure that a given {@link Collection} is not in a persistent state. + * + * @param objects The {@link Collection} to check the state of + * @param message Message to use for the exception, if object is persistent + * @see #assertNonPersistent(Object, String) + * @since 4.11.0 + */ + public static void assertNonPersistentAll(final Collection objects, final String message) { + if (objects == null || objects.isEmpty()) { + return; + } + + objects.forEach(object -> assertNonPersistent(object, message)); + } + private static boolean isPersistent(final Object object) { final ObjectState objectState = JDOHelper.getObjectState(object); return objectState == PERSISTENT_CLEAN