fix: stream NVD data via Jackson to reduce memory footprint (#6275)

core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/CveApiJson20CveItemSource.java

@@ -0,0 +1,88 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update.nvd.api;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import io.github.jeremylong.openvulnerability.client.nvd.DefCveItem;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.util.zip.GZIPInputStream;
+
+public class CveApiJson20CveItemSource implements CveItemSource<DefCveItem> {
+
+    private final File jsonFile;
+    private final ObjectMapper mapper;
+    private final InputStream inputStream;
+    private final JsonParser jsonParser;
+    private DefCveItem currentItem;
+    private DefCveItem nextItem;
+
+    public CveApiJson20CveItemSource(File jsonFile) throws IOException {
+        this.jsonFile = jsonFile;
+        mapper = new ObjectMapper();
+        mapper.registerModule(new JavaTimeModule());
+        inputStream = jsonFile.getName().endsWith(".gz") ?
+                new GZIPInputStream(new BufferedInputStream(Files.newInputStream(jsonFile.toPath()))) :
+                new BufferedInputStream(Files.newInputStream(jsonFile.toPath()));
+        jsonParser = mapper.getFactory().createParser(inputStream);
+
+        JsonToken token = null;
+        do {
+            token = jsonParser.nextToken();
+            if (token  == JsonToken.FIELD_NAME) {
+                String fieldName = jsonParser.getCurrentName();
+                if (fieldName.equals("vulnerabilities") && (jsonParser.nextToken() == JsonToken.START_ARRAY)) {
+                    nextItem = readItem(jsonParser);
+                }
+            }
+        } while (token != null && nextItem == null);
+    }
+
+    @Override
+    public void close() throws Exception {
+        jsonParser.close();
+        inputStream.close();
+        Files.delete(jsonFile.toPath());
+    }
+
+    @Override
+    public boolean hasNext() {
+        return nextItem != null;
+    }
+
+    @Override
+    public DefCveItem next() throws IOException {
+        currentItem = nextItem;
+        nextItem = readItem(jsonParser);
+        return currentItem;
+    }
+
+    private DefCveItem readItem(JsonParser jsonParser) throws IOException {
+        if (jsonParser.nextToken() == JsonToken.START_OBJECT) {
+            return mapper.readValue(jsonParser, DefCveItem.class);
+        }
+        return null;
+    }
+}

core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/CveItemSource.java

@@ -0,0 +1,29 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update.nvd.api;
+
+import io.github.jeremylong.openvulnerability.client.nvd.DefCveItem;
+
+import java.io.IOException;
+
+public interface CveItemSource<T extends DefCveItem> extends AutoCloseable {
+
+    boolean hasNext();
+
+    T next() throws IOException;
+}

core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/JsonArrayCveItemSource.java

@@ -0,0 +1,81 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update.nvd.api;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import io.github.jeremylong.openvulnerability.client.nvd.DefCveItem;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.util.zip.GZIPInputStream;
+
+public class JsonArrayCveItemSource implements CveItemSource<DefCveItem> {
+
+    private final File jsonFile;
+    private final ObjectMapper mapper;
+    private final InputStream inputStream;
+    private final JsonParser jsonParser;
+    private DefCveItem currentItem;
+    private DefCveItem nextItem;
+
+    public JsonArrayCveItemSource(File jsonFile) throws IOException {
+        this.jsonFile = jsonFile;
+        mapper = new ObjectMapper();
+        mapper.registerModule(new JavaTimeModule());
+        inputStream = jsonFile.getName().endsWith(".gz") ?
+                new GZIPInputStream(new BufferedInputStream(Files.newInputStream(jsonFile.toPath()))) :
+                new BufferedInputStream(Files.newInputStream(jsonFile.toPath()));
+        jsonParser = mapper.getFactory().createParser(inputStream);
+
+        if (jsonParser.nextToken() == JsonToken.START_ARRAY) {
+            nextItem = readItem(jsonParser);
+        }
+    }
+
+    @Override
+    public void close() throws Exception {
+        jsonParser.close();
+        inputStream.close();
+        Files.delete(jsonFile.toPath());
+    }
+
+    @Override
+    public boolean hasNext() {
+        return nextItem != null;
+    }
+
+    @Override
+    public DefCveItem next() throws IOException {
+        currentItem = nextItem;
+        nextItem = readItem(jsonParser);
+        return currentItem;
+    }
+
+    private DefCveItem readItem(JsonParser jsonParser) throws IOException {
+        if (jsonParser.nextToken() == JsonToken.START_OBJECT) {
+            return mapper.readValue(jsonParser, DefCveItem.class);
+        }
+        return null;
+    }
+}

core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/NvdApiProcessor.java

@@ -17,20 +17,11 @@
  */
 package org.owasp.dependencycheck.data.update.nvd.api;
 
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
-import io.github.jeremylong.openvulnerability.client.nvd.CveApiJson20;
 import io.github.jeremylong.openvulnerability.client.nvd.DefCveItem;
 import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.util.Collection;
 import java.util.concurrent.Callable;
-import java.util.zip.GZIPInputStream;
 import org.owasp.dependencycheck.data.nvd.ecosystem.CveEcosystemMapper;
 import org.owasp.dependencycheck.data.nvdcve.CveDB;
-import org.owasp.dependencycheck.data.update.exception.UpdateException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -91,38 +82,26 @@ public NvdApiProcessor(final CveDB cveDB, File jsonFile) {
 
     @Override
     public NvdApiProcessor call() throws Exception {
-        final ObjectMapper objectMapper = new ObjectMapper();
-        objectMapper.registerModule(new JavaTimeModule());
-         Collection<DefCveItem> data = null;
+        CveItemSource<DefCveItem> itemSource = null;
 
-         if (jsonFile.getName().endsWith(".jsonarray.gz")) {
-            try (FileInputStream fileInputStream = new FileInputStream(jsonFile);
-                        GZIPInputStream gzipInputStream = new GZIPInputStream(fileInputStream);) {
-                data = objectMapper.readValue(gzipInputStream, new TypeReference<Collection<DefCveItem>>(){});
-            } catch (IOException exception) {
-                throw new UpdateException("Unable to read downloaded json data: " + jsonFile, exception);
-            }
+        if (jsonFile.getName().endsWith(".jsonarray.gz")) {
+            itemSource = new JsonArrayCveItemSource(jsonFile);
         } else if (jsonFile.getName().endsWith(".gz")) {
-            try (FileInputStream fileInputStream = new FileInputStream(jsonFile);
-                        GZIPInputStream gzipInputStream = new GZIPInputStream(fileInputStream);) {
-                CveApiJson20 cveData = objectMapper.readValue(gzipInputStream, CveApiJson20.class);
-                if (cveData != null) {
-                    data = cveData.getVulnerabilities();
-                }
-            } catch (IOException exception) {
-                throw new UpdateException("Unable to read downloaded json data: " + jsonFile, exception);
-            }
+            itemSource = new CveApiJson20CveItemSource(jsonFile);
         } else {
-            data = objectMapper.readValue(jsonFile, new TypeReference<Collection<DefCveItem>>(){});
+            itemSource = new JsonArrayCveItemSource(jsonFile);
         }
-        if (data != null ) {
-            for (DefCveItem entry : data) {
+        try {
+            while (itemSource.hasNext()) {
+                DefCveItem entry = itemSource.next();
                 try {
                     cveDB.updateVulnerability(entry, mapper.getEcosystem(entry));
                 } catch (Exception ex) {
                     LOGGER.error("Failed to process " + entry.getCve().getId(), ex);
                 }
             }
+        } finally {
+            itemSource.close();
         }
         endTime = System.currentTimeMillis();
         return this;