Restore lucene index during startup & allow rebuild through UI (Depen…

…dencyTrack#2200)

* Fix: Restoring lucene index build during startup by having a dedicated listener

A REST API is also exposed to allow index rebuild through the GUI. See DependencyTrack#2104
Automatic periodic consistency check with database are performed if enabled

Signed-off-by: Alioune SY <[email protected]>

* Fix: Restoring lucene index build during startup by having a dedicated listener

Takint into account review comments

Signed-off-by: Alioune SY <[email protected]>

* Fix: Restoring lucene index build during startup by having a dedicated listener

Fixing unit tests.

Signed-off-by: Alioune SY <[email protected]>

Signed-off-by: Alioune SY <[email protected]>

Fixes DependencyTrack#2104

Signed-off-by: mulder999 <[email protected]>

src/main/java/org/dependencytrack/event/IndexEvent.java

@@ -18,72 +18,78 @@
  */
 package org.dependencytrack.event;
 
-import alpine.event.framework.Event;
+import alpine.event.framework.AbstractChainableEvent;
+import alpine.event.framework.SingletonCapableEvent;
 import org.dependencytrack.model.Component;
 import org.dependencytrack.model.Cpe;
 import org.dependencytrack.model.License;
 import org.dependencytrack.model.Project;
 import org.dependencytrack.model.ServiceComponent;
 import org.dependencytrack.model.Vulnerability;
 import org.dependencytrack.model.VulnerableSoftware;
+import org.dependencytrack.search.IndexManager;
 
 /**
  * Defines various Lucene index events.
  *
  * @author Steve Springett
  * @since 3.0.0
  */
-public class IndexEvent implements Event {
+public class IndexEvent extends SingletonCapableEvent {
 
     public enum Action {
         CREATE,
         UPDATE,
         DELETE,
         COMMIT,
-        REINDEX
+        REINDEX,
+        CHECK
     }
 
     private final Action action;
     private Object indexableObject;
     private Class indexableClass;
 
-
     public IndexEvent(final Action action, final Project project) {
-        this.action = action;
+        this(action, Project.class);
         this.indexableObject = project;
     }
 
     public IndexEvent(final Action action, final Component component) {
-        this.action = action;
+        this(action, Component.class);
         this.indexableObject = component;
     }
 
     public IndexEvent(final Action action, final ServiceComponent service) {
-        this.action = action;
+        this(action, ServiceComponent.class);
         this.indexableObject = service;
     }
 
     public IndexEvent(final Action action, final Vulnerability vulnerability) {
-        this.action = action;
+        this(action, Vulnerability.class);
         this.indexableObject = vulnerability;
     }
 
     public IndexEvent(final Action action, final License license) {
-        this.action = action;
+        this(action, License.class);
         this.indexableObject = license;
     }
 
     public IndexEvent(final Action action, final Cpe cpe) {
-        this.action = action;
+        this(action, Cpe.class);
         this.indexableObject = cpe;
     }
 
     public IndexEvent(final Action action, final VulnerableSoftware vs) {
-        this.action = action;
+        this(action, VulnerableSoftware.class);
         this.indexableObject = vs;
     }
 
     public IndexEvent(final Action action, final Class clazz) {
+        if(action == Action.REINDEX) {
+            this.setSingleton(true);
+            this.setChainIdentifier(IndexManager.IndexType.getUuid(clazz));
+        }
         this.action = action;
         this.indexableClass = clazz;
     }

src/main/java/org/dependencytrack/model/ConfigPropertyConstants.java

@@ -90,7 +90,10 @@ public enum ConfigPropertyConstants {
     TASK_SCHEDULER_PORTFOLIO_VULNERABILITY_ANALYSIS_CADENCE("task-scheduler", "portfolio.vulnerability.analysis.cadence", "24", PropertyType.INTEGER, "Launch cadence (in hours) for portfolio vulnerability analysis"),
     TASK_SCHEDULER_REPOSITORY_METADATA_FETCH_CADENCE("task-scheduler", "repository.metadata.fetch.cadence", "24", PropertyType.INTEGER, "Metadada fetch cadence (in hours) for package repositories"),
     TASK_SCHEDULER_INTERNAL_COMPONENT_IDENTIFICATION_CADENCE("task-scheduler", "internal.components.identification.cadence", "6", PropertyType.INTEGER, "Internal component identification cadence (in hours)"),
-    TASK_SCHEDULER_COMPONENT_ANALYSIS_CACHE_CLEAR_CADENCE("task-scheduler", "component.analysis.cache.clear.cadence", "24", PropertyType.INTEGER, "Cleanup cadence (in hours) for component analysis cache");
+    TASK_SCHEDULER_COMPONENT_ANALYSIS_CACHE_CLEAR_CADENCE("task-scheduler", "component.analysis.cache.clear.cadence", "24", PropertyType.INTEGER, "Cleanup cadence (in hours) for component analysis cache"),
+    SEARCH_INDEXES_CONSISTENCY_CHECK_ENABLED("search-indexes", "consistency.check.enabled", "true", PropertyType.BOOLEAN, "Flag to enable lucene indexes periodic consistency check"),
+    SEARCH_INDEXES_CONSISTENCY_CHECK_CADENCE("search-indexes", "consistency.check.cadence", "4320", PropertyType.INTEGER, "Lucene indexes consistency check cadence (in minutes)"),
+    SEARCH_INDEXES_CONSISTENCY_CHECK_DELTA_THRESHOLD("search-indexes", "consistency.check.delta.threshold", "20", PropertyType.INTEGER, "Threshold used to trigger an index rebuild when comparing database table and corresponding lucene index (in percentage). It must be an integer between 1 and 100");
 
     private String groupName;
     private String propertyName;

src/main/java/org/dependencytrack/persistence/DefaultObjectGenerator.java

@@ -66,27 +66,6 @@ public void contextInitialized(final ServletContextEvent event) {
             return;
         }
 
-        if (!IndexManager.exists(IndexManager.IndexType.LICENSE)) {
-            LOGGER.info("Dispatching event to reindex licenses");
-            Event.dispatch(new IndexEvent(IndexEvent.Action.REINDEX, License.class));
-        }
-        if (!IndexManager.exists(IndexManager.IndexType.PROJECT)) {
-            LOGGER.info("Dispatching event to reindex projects");
-            Event.dispatch(new IndexEvent(IndexEvent.Action.REINDEX, Project.class));
-        }
-        if (!IndexManager.exists(IndexManager.IndexType.COMPONENT)) {
-            LOGGER.info("Dispatching event to reindex components");
-            Event.dispatch(new IndexEvent(IndexEvent.Action.REINDEX, Component.class));
-        }
-        if (!IndexManager.exists(IndexManager.IndexType.VULNERABILITY)) {
-            LOGGER.info("Dispatching event to reindex vulnerabilities");
-            Event.dispatch(new IndexEvent(IndexEvent.Action.REINDEX, Vulnerability.class));
-        }
-        if (!IndexManager.exists(IndexManager.IndexType.VULNERABLESOFTWARE)) {
-            LOGGER.info("Dispatching event to reindex vulnerablesoftware");
-            Event.dispatch(new IndexEvent(IndexEvent.Action.REINDEX, VulnerableSoftware.class));
-        }
-
         loadDefaultPermissions();
         loadDefaultPersonas();
         loadDefaultLicenses();

src/main/java/org/dependencytrack/resources/v1/AbstractConfigPropertyResource.java

@@ -63,6 +63,9 @@ private Response updatePropertyValueInternal(IConfigProperty json, IConfigProper
                 if(ConfigPropertyConstants.TASK_SCHEDULER_LDAP_SYNC_CADENCE.getGroupName().equals(json.getGroupName()) && propertyValue <= 0) {
                     return Response.status(Response.Status.BAD_REQUEST).entity("A Task scheduler cadence ("+json.getPropertyName()+") cannot be inferior to one hour.A value of "+propertyValue+" was provided.").build();
                 }
+                if(ConfigPropertyConstants.SEARCH_INDEXES_CONSISTENCY_CHECK_DELTA_THRESHOLD.getPropertyName().equals(json.getPropertyName()) && (propertyValue < 1 || propertyValue > 100)) {
+                    return Response.status(Response.Status.BAD_REQUEST).entity("Lucene index delta threshold ("+json.getPropertyName()+") cannot be inferior to 1 or superior to 100.A value of "+propertyValue+" was provided.").build();
+                }
                 property.setPropertyValue(String.valueOf(propertyValue));
             } catch (NumberFormatException e) {
                 return Response.status(Response.Status.BAD_REQUEST).entity("The property expected an integer and an integer was not sent.").build();

src/main/java/org/dependencytrack/resources/v1/SearchResource.java

@@ -32,11 +32,15 @@
 import org.dependencytrack.search.SearchResult;
 
 import javax.ws.rs.GET;
+import javax.ws.rs.POST;
 import javax.ws.rs.Path;
 import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
 
 /**
  * JAX-RS resources for processing search requests.
@@ -177,4 +181,28 @@ public Response vulnerableSoftwareSearch(@QueryParam("query") String query, @Que
             return Response.ok(searchResult).build();
         }
     }
+
+    @Path("/reindex")
+    @POST
+    @Produces(MediaType.APPLICATION_JSON)
+    @ApiOperation(
+            value = "Rebuild lucene indexes for search operations"
+    )
+    @ApiResponses(value = {
+            @ApiResponse(code = 401, message = "Unauthorized"),
+            @ApiResponse(code = 400, message = "No valid index type was provided")
+    })
+    @PermissionRequired(Permissions.Constants.SYSTEM_CONFIGURATION)
+    public Response reindex(@QueryParam("type") Set<String> type) {
+        if (type == null || type.isEmpty()) {
+            return Response.status(Response.Status.BAD_REQUEST).entity("No valid index type was provided").build();
+        }
+        try {
+            final SearchManager searchManager = new SearchManager();
+            String chainIdentifier = searchManager.reindex(type);
+            return Response.ok(Collections.singletonMap("token", chainIdentifier)).build();
+        } catch (IllegalArgumentException exception) {
+            return Response.status(Response.Status.BAD_REQUEST).entity(exception.getMessage()).build();
+        }
+    }
 }

src/main/java/org/dependencytrack/search/ComponentIndexer.java

@@ -23,6 +23,7 @@
 import alpine.notification.NotificationLevel;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.Term;
 import org.dependencytrack.model.Component;
 import org.dependencytrack.model.Project;
@@ -77,6 +78,8 @@ public void add(final Component component) {
 
         try {
             getIndexWriter().addDocument(doc);
+        } catch (CorruptIndexException e) {
+            handleCorruptIndexException(e);
         } catch (IOException e) {
             LOGGER.error("An error occurred while adding component to index", e);
             Notification.dispatch(new Notification()
@@ -97,6 +100,8 @@ public void add(final Component component) {
     public void remove(final Component component) {
         try {
             getIndexWriter().deleteDocuments(new Term(IndexConstants.COMPONENT_UUID, component.getUuid().toString()));
+        } catch (CorruptIndexException e) {
+            handleCorruptIndexException(e);
         } catch (IOException e) {
             LOGGER.error("An error occurred while removing a component from the index", e);
             Notification.dispatch(new Notification()

src/main/java/org/dependencytrack/search/CpeIndexer.java

@@ -24,6 +24,7 @@
 import alpine.persistence.PaginatedResult;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.Term;
 import org.dependencytrack.model.Cpe;
 import org.dependencytrack.notification.NotificationConstants;
@@ -77,6 +78,8 @@ public void add(final Cpe cpe) {
 
         try {
             getIndexWriter().addDocument(doc);
+        } catch (CorruptIndexException e) {
+            handleCorruptIndexException(e);
         } catch (IOException e) {
             LOGGER.error("An error occurred while adding a CPE to the index", e);
             Notification.dispatch(new Notification()
@@ -97,6 +100,8 @@ public void add(final Cpe cpe) {
     public void remove(final Cpe cpe) {
         try {
             getIndexWriter().deleteDocuments(new Term(IndexConstants.CPE_UUID, cpe.getUuid().toString()));
+        } catch (CorruptIndexException e) {
+            handleCorruptIndexException(e);
         } catch (IOException e) {
             LOGGER.error("An error occurred while removing a CPE from the index", e);
             Notification.dispatch(new Notification()

src/main/java/org/dependencytrack/search/FuzzyVulnerableSoftwareSearchManager.java

@@ -1,6 +1,7 @@
 package org.dependencytrack.search;
 
 import alpine.common.logging.Logger;
+import alpine.event.framework.Event;
 import alpine.notification.Notification;
 import alpine.notification.NotificationLevel;
 import com.google.common.collect.Sets;
@@ -13,6 +14,7 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TopDocs;
+import org.dependencytrack.event.IndexEvent;
 import org.dependencytrack.model.Component;
 import org.dependencytrack.model.VulnerableSoftware;
 import org.dependencytrack.notification.NotificationConstants;
@@ -171,6 +173,8 @@ public SearchResult searchIndex(final String luceneQuery) {
                     .content("Corrupted Lucene index detected. Check log for details. " + e.getMessage())
                     .level(NotificationLevel.ERROR)
             );
+            LOGGER.info("Trying to rebuild the corrupted index "+indexManager.getIndexType().name());
+            Event.dispatch(new IndexEvent(IndexEvent.Action.REINDEX, indexManager.getIndexType().getClazz()));
         } catch (IOException e) {
             LOGGER.error("An I/O Exception occurred while searching Lucene index", e);
             Notification.dispatch(new Notification()

src/main/java/org/dependencytrack/search/IndexConstants.java

@@ -80,7 +80,7 @@ public final class IndexConstants {
     static final String CPE_PRODUCT = "product";
     static final String CPE_VERSION = "version";
     static final String[] CPE_SEARCH_FIELDS = {
-            CPE_22, CPE_23, CPE_VENDOR, CPE_PRODUCT, CPE_VERSION
+            CPE_UUID, CPE_22, CPE_23, CPE_VENDOR, CPE_PRODUCT, CPE_VERSION
     };
 
     static final String VULNERABLESOFTWARE_UUID = "uuid";
@@ -90,7 +90,7 @@ public final class IndexConstants {
     static final String VULNERABLESOFTWARE_PRODUCT = "product";
     static final String VULNERABLESOFTWARE_VERSION = "version";
     static final String[] VULNERABLESOFTWARE_SEARCH_FIELDS = {
-            VULNERABLESOFTWARE_CPE_22, VULNERABLESOFTWARE_CPE_23, VULNERABLESOFTWARE_VENDOR,
+            VULNERABILITY_UUID, VULNERABLESOFTWARE_CPE_22, VULNERABLESOFTWARE_CPE_23, VULNERABLESOFTWARE_VENDOR,
             VULNERABLESOFTWARE_PRODUCT, VULNERABLESOFTWARE_VERSION
     };