Added Index Factory Config (open-metadata#14950)

* Index Factory Config * CI handling * checkstyle
Abhishek332 · Feb 1, 2024 · 795bd5c · 795bd5c
1 parent a4f3ed8
commit 795bd5c
Show file tree

Hide file tree

Showing 7 changed files with 26 additions and 15 deletions.
diff --git a/conf/openmetadata.yaml b/conf/openmetadata.yaml
@@ -256,7 +256,8 @@ elasticsearch:
   socketTimeoutSecs: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
   keepAliveTimeoutSecs: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
   batchSize: ${ELASTICSEARCH_BATCH_SIZE:-10}
-  searchIndexMappingLanguage: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
+  searchIndexMappingLanguage : ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
+  searchIndexFactoryClassName : org.openmetadata.service.search.SearchIndexFactory
 
 eventMonitoringConfiguration:
   eventMonitor: ${EVENT_MONITOR:-prometheus}  # Possible values are "prometheus", "cloudwatch"

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java
@@ -90,7 +90,6 @@
 import org.openmetadata.service.resources.CollectionRegistry;
 import org.openmetadata.service.resources.databases.DatasourceConfig;
 import org.openmetadata.service.resources.settings.SettingsCache;
-import org.openmetadata.service.search.SearchIndexFactory;
 import org.openmetadata.service.search.SearchRepository;
 import org.openmetadata.service.secrets.SecretsManager;
 import org.openmetadata.service.secrets.SecretsManagerFactory;
@@ -146,7 +145,7 @@ public void run(OpenMetadataApplicationConfig catalogConfig, Environment environ
 
     // initialize Search Repository, all repositories use SearchRepository this line should always
     // before initializing repository
-    new SearchRepository(catalogConfig.getElasticSearchConfiguration(), new SearchIndexFactory());
+    new SearchRepository(catalogConfig.getElasticSearchConfiguration());
     // as first step register all the repositories
     Entity.initializeRepositories(catalogConfig, jdbi);
 

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java
@@ -75,7 +75,6 @@
 import org.openmetadata.service.jdbi3.ListFilter;
 import org.openmetadata.service.resources.Collection;
 import org.openmetadata.service.resources.EntityResource;
-import org.openmetadata.service.search.SearchIndexFactory;
 import org.openmetadata.service.search.SearchRepository;
 import org.openmetadata.service.secrets.SecretsManager;
 import org.openmetadata.service.secrets.SecretsManagerFactory;
@@ -114,8 +113,7 @@ public void initialize(OpenMetadataApplicationConfig config) {
 
     // Create an On Demand DAO
     CollectionDAO dao = Entity.getCollectionDAO();
-    searchRepository =
-        new SearchRepository(config.getElasticSearchConfiguration(), new SearchIndexFactory());
+    searchRepository = new SearchRepository(config.getElasticSearchConfiguration());
 
     try {
       AppScheduler.initialize(dao, searchRepository);

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java
@@ -92,16 +92,28 @@ public class SearchRepository {
 
   public static final String ELASTIC_SEARCH_EXTENSION = "service.eventPublisher";
 
-  public SearchRepository(
-      ElasticSearchConfiguration config, SearchIndexFactory searchIndexFactory) {
+  public SearchRepository(ElasticSearchConfiguration config) {
     elasticSearchConfiguration = config;
     if (config != null
         && config.getSearchType() == ElasticSearchConfiguration.SearchType.OPENSEARCH) {
       searchClient = new OpenSearchClient(config);
     } else {
       searchClient = new ElasticSearchClient(config);
     }
-    this.searchIndexFactory = searchIndexFactory;
+    try {
+      if (config != null) {
+        this.searchIndexFactory =
+            Class.forName(config.getSearchIndexFactoryClassName())
+                .asSubclass(SearchIndexFactory.class)
+                .newInstance();
+      }
+    } catch (ClassNotFoundException e) {
+      LOG.warn("Failed to initialize search index factory", e);
+    } catch (InstantiationException e) {
+      LOG.warn("Failed to initialize search index factory", e);
+    } catch (IllegalAccessException e) {
+      LOG.warn("Failed to initialize search index factory", e);
+    }
     language =
         config != null && config.getSearchIndexMappingLanguage() != null
             ? config.getSearchIndexMappingLanguage().value()

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java
@@ -57,7 +57,6 @@
 import org.openmetadata.service.jdbi3.locator.ConnectionType;
 import org.openmetadata.service.migration.api.MigrationWorkflow;
 import org.openmetadata.service.resources.databases.DatasourceConfig;
-import org.openmetadata.service.search.SearchIndexFactory;
 import org.openmetadata.service.search.SearchRepository;
 import org.openmetadata.service.secrets.SecretsManager;
 import org.openmetadata.service.secrets.SecretsManagerFactory;
@@ -392,8 +391,7 @@ private void parseConfig() throws Exception {
             new ConnectionAwareAnnotationSqlLocator(
                 config.getDataSourceFactory().getDriverClass()));
 
-    searchRepository =
-        new SearchRepository(config.getElasticSearchConfiguration(), new SearchIndexFactory());
+    searchRepository = new SearchRepository(config.getElasticSearchConfiguration());
 
     // Initialize secrets manager
     secretsManager =

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/TablesInitializer.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/TablesInitializer.java
@@ -50,7 +50,6 @@
 import org.openmetadata.service.jdbi3.locator.ConnectionType;
 import org.openmetadata.service.migration.api.MigrationWorkflow;
 import org.openmetadata.service.resources.databases.DatasourceConfig;
-import org.openmetadata.service.search.SearchIndexFactory;
 import org.openmetadata.service.search.SearchRepository;
 import org.openmetadata.service.secrets.SecretsManagerFactory;
 import org.openmetadata.service.util.jdbi.DatabaseAuthenticationProviderFactory;
@@ -299,7 +298,7 @@ private static void execute(
             new ConnectionAwareAnnotationSqlLocator(
                 config.getDataSourceFactory().getDriverClass()));
     SearchRepository searchRepository =
-        new SearchRepository(config.getElasticSearchConfiguration(), new SearchIndexFactory());
+        new SearchRepository(config.getElasticSearchConfiguration());
 
     // Initialize secrets manager
     SecretsManagerFactory.createSecretsManager(
@@ -396,7 +395,7 @@ public static void validateAndRunSystemDataMigrations(
         new MigrationWorkflow(
             jdbi, nativeMigrationSQLPath, connType, extensionSQLScriptRootPath, forceMigrations);
     // Initialize search repository
-    new SearchRepository(config.getElasticSearchConfiguration(), new SearchIndexFactory());
+    new SearchRepository(config.getElasticSearchConfiguration());
     Entity.setCollectionDAO(jdbi.onDemand(CollectionDAO.class));
     Entity.initializeRepositories(config, jdbi);
     workflow.loadMigrations();

diff --git a/...etadata-spec/src/main/resources/json/schema/configuration/elasticSearchConfiguration.json b/...etadata-spec/src/main/resources/json/schema/configuration/elasticSearchConfiguration.json
@@ -82,6 +82,10 @@
         "opensearch"
       ],
       "default": "elasticsearch"
+    },
+    "searchIndexFactoryClassName": {
+      "description": "Index factory name",
+      "type": "string"
     }
   },
   "required": ["host", "port", "scheme", "connectionTimeoutSecs", "socketTimeoutSecs", "batchSize", "searchIndexMappingLanguage"],