Datasources Integ Test Fix

integ-test/build.gradle

@@ -124,7 +124,6 @@ testClusters.all {
 
 testClusters.integTest {
     plugin ":opensearch-sql-plugin"
-    keystore 'plugins.query.federation.datasources.config', new File("$projectDir/src/test/resources/datasource/", 'datasources.json')
 }
 
 task startPrometheus(type: SpawnProcessTask) {

integ-test/src/test/java/org/opensearch/sql/datasource/DataSourceAPIsIT.java

@@ -12,12 +12,15 @@
 import com.google.gson.Gson;
 import com.google.gson.JsonObject;
 import com.google.gson.reflect.TypeToken;
+import java.io.IOException;
 import java.lang.reflect.Type;
 import java.util.ArrayList;
 import java.util.List;
 import lombok.SneakyThrows;
 import org.apache.commons.lang3.StringUtils;
+import org.junit.AfterClass;
 import org.junit.Assert;
+import org.junit.BeforeClass;
 import org.junit.Test;
 import org.opensearch.client.Request;
 import org.opensearch.client.RequestOptions;
@@ -29,6 +32,21 @@
 
 public class DataSourceAPIsIT extends PPLIntegTestCase {
 
+  @AfterClass
+  protected static void deleteDataSourcesCreated() throws IOException {
+    Request deleteRequest = getDeleteDataSourceRequest("create_prometheus");
+    Response deleteResponse = client().performRequest(deleteRequest);
+    Assert.assertEquals(204, deleteResponse.getStatusLine().getStatusCode());
+
+    deleteRequest = getDeleteDataSourceRequest("update_prometheus");
+    deleteResponse = client().performRequest(deleteRequest);
+    Assert.assertEquals(204, deleteResponse.getStatusLine().getStatusCode());
+
+    deleteRequest = getDeleteDataSourceRequest("get_all_prometheus");
+    deleteResponse = client().performRequest(deleteRequest);
+    Assert.assertEquals(204, deleteResponse.getStatusLine().getStatusCode());
+  }
+
   @SneakyThrows
   @Test
   public void createDataSourceAPITest() {
@@ -150,43 +168,4 @@ public void getAllDataSourceTest() {
         dataSourceMetadataList.stream().anyMatch(ds -> ds.getName().equals("get_all_prometheus")));
   }
 
-
-    private Request getCreateDataSourceRequest(DataSourceMetadata dataSourceMetadata) {
-    Request request = new Request("POST", "/_plugins/_query/_datasources");
-    request.setJsonEntity(new Gson().toJson(dataSourceMetadata));
-    RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder();
-    restOptionsBuilder.addHeader("Content-Type", "application/json");
-    request.setOptions(restOptionsBuilder);
-    return request;
-  }
-
-  private Request getUpdateDataSourceRequest(DataSourceMetadata dataSourceMetadata) {
-    Request request = new Request("PUT", "/_plugins/_query/_datasources");
-    request.setJsonEntity(new Gson().toJson(dataSourceMetadata));
-    RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder();
-    restOptionsBuilder.addHeader("Content-Type", "application/json");
-    request.setOptions(restOptionsBuilder);
-    return request;
-  }
-
-  private Request getFetchDataSourceRequest(String name) {
-    Request request = new Request("GET", "/_plugins/_query/_datasources" + "/" + name);
-    if (StringUtils.isEmpty(name)) {
-      request = new Request("GET", "/_plugins/_query/_datasources");
-    }
-    RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder();
-    restOptionsBuilder.addHeader("Content-Type", "application/json");
-    request.setOptions(restOptionsBuilder);
-    return request;
-  }
-
-
-  private Request getDeleteDataSourceRequest(String name) {
-    Request request = new Request("DELETE", "/_plugins/_query/_datasources" + "/" + name);
-    RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder();
-    restOptionsBuilder.addHeader("Content-Type", "application/json");
-    request.setOptions(restOptionsBuilder);
-    return request;
-  }
-
 }

integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java

@@ -7,6 +7,8 @@
 package org.opensearch.sql.legacy;
 
 import com.google.common.base.Strings;
+import com.google.gson.Gson;
+import org.apache.commons.lang3.StringUtils;
 import org.json.JSONArray;
 import org.json.JSONObject;
 import org.junit.AfterClass;
@@ -30,6 +32,7 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Locale;
+import org.opensearch.sql.datasource.model.DataSourceMetadata;
 
 import static com.google.common.base.Strings.isNullOrEmpty;
 import static org.opensearch.sql.legacy.TestUtils.createIndexByRestClient;
@@ -441,6 +444,44 @@ protected JSONObject getSource(JSONObject hit) {
     return hit.getJSONObject("_source");
   }
 
+  protected static Request getCreateDataSourceRequest(DataSourceMetadata dataSourceMetadata) {
+    Request request = new Request("POST", "/_plugins/_query/_datasources");
+    request.setJsonEntity(new Gson().toJson(dataSourceMetadata));
+    RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder();
+    restOptionsBuilder.addHeader("Content-Type", "application/json");
+    request.setOptions(restOptionsBuilder);
+    return request;
+  }
+
+  protected static Request getUpdateDataSourceRequest(DataSourceMetadata dataSourceMetadata) {
+    Request request = new Request("PUT", "/_plugins/_query/_datasources");
+    request.setJsonEntity(new Gson().toJson(dataSourceMetadata));
+    RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder();
+    restOptionsBuilder.addHeader("Content-Type", "application/json");
+    request.setOptions(restOptionsBuilder);
+    return request;
+  }
+
+  protected static Request getFetchDataSourceRequest(String name) {
+    Request request = new Request("GET", "/_plugins/_query/_datasources" + "/" + name);
+    if (StringUtils.isEmpty(name)) {
+      request = new Request("GET", "/_plugins/_query/_datasources");
+    }
+    RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder();
+    restOptionsBuilder.addHeader("Content-Type", "application/json");
+    request.setOptions(restOptionsBuilder);
+    return request;
+  }
+
+
+  protected static Request getDeleteDataSourceRequest(String name) {
+    Request request = new Request("DELETE", "/_plugins/_query/_datasources" + "/" + name);
+    RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder();
+    restOptionsBuilder.addHeader("Content-Type", "application/json");
+    request.setOptions(restOptionsBuilder);
+    return request;
+  }
+
   /**
    * Enum for associating test index with relevant mapping and data.
    */
@@ -637,5 +678,7 @@ public String getMapping() {
     public String getDataSet() {
       return this.dataSet;
     }
+
+
   }
 }

integ-test/src/test/java/org/opensearch/sql/ppl/InformationSchemaCommandIT.java

@@ -7,28 +7,31 @@
 
 package org.opensearch.sql.ppl;
 
+import static org.opensearch.sql.legacy.TestUtils.getResponseBody;
 import static org.opensearch.sql.util.MatcherUtils.columnName;
 import static org.opensearch.sql.util.MatcherUtils.rows;
 import static org.opensearch.sql.util.MatcherUtils.verifyColumn;
 import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;
 
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import java.io.IOException;
 import org.json.JSONObject;
 import org.junit.After;
 import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
+import org.opensearch.client.Request;
+import org.opensearch.client.Response;
+import org.opensearch.sql.datasource.model.DataSourceMetadata;
+import org.opensearch.sql.datasource.model.DataSourceType;
 
 public class InformationSchemaCommandIT extends PPLIntegTestCase {
 
-
-  @Override
-  protected void init() throws Exception {
-    loadIndex(Index.DATASOURCES);
-  }
-
   /**
    * Integ tests are dependent on self generated metrics in prometheus instance.
    * When running individual integ tests there
@@ -41,6 +44,23 @@ protected static void metricGenerationWait() throws InterruptedException {
     Thread.sleep(10000);
   }
 
+  @Override
+  protected void init() throws InterruptedException, IOException {
+    DataSourceMetadata createDSM =
+        new DataSourceMetadata("my_prometheus", DataSourceType.PROMETHEUS,
+            ImmutableList.of(), ImmutableMap.of("prometheus.uri", "http://localhost:9090"));
+    Request createRequest = getCreateDataSourceRequest(createDSM);
+    Response response = client().performRequest(createRequest);
+    Assert.assertEquals(201, response.getStatusLine().getStatusCode());
+  }
+
+  @After
+  protected void deleteDataSourceMetadata() throws IOException {
+    Request deleteRequest = getDeleteDataSourceRequest("my_prometheus");
+    Response deleteResponse = client().performRequest(deleteRequest);
+    Assert.assertEquals(204, deleteResponse.getStatusLine().getStatusCode());
+  }
+
   @Test
   public void testSearchTablesFromPrometheusCatalog() throws IOException {
     JSONObject result =

integ-test/src/test/java/org/opensearch/sql/ppl/PrometheusDataSourceCommandsIT.java

@@ -12,24 +12,28 @@
 import static org.opensearch.sql.util.MatcherUtils.schema;
 import static org.opensearch.sql.util.MatcherUtils.verifySchema;
 
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import java.io.IOException;
 import java.text.SimpleDateFormat;
 import java.util.Date;
 import lombok.SneakyThrows;
 import org.apache.commons.lang3.StringUtils;
 import org.json.JSONArray;
 import org.json.JSONObject;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
+import org.opensearch.client.Request;
+import org.opensearch.client.Response;
+import org.opensearch.sql.datasource.model.DataSourceMetadata;
+import org.opensearch.sql.datasource.model.DataSourceType;
 
 public class PrometheusDataSourceCommandsIT extends PPLIntegTestCase {
 
-
-  @Override
-  protected void init() throws Exception {
-    loadIndex(Index.DATASOURCES);
-  }
-
   /**
    * Integ tests are dependent on self generated metrics in prometheus instance.
    * When running individual integ tests there
@@ -42,6 +46,23 @@ protected static void metricGenerationWait() throws InterruptedException {
     Thread.sleep(10000);
   }
 
+  @Override
+  protected void init() throws InterruptedException, IOException {
+    DataSourceMetadata createDSM =
+        new DataSourceMetadata("my_prometheus", DataSourceType.PROMETHEUS,
+            ImmutableList.of(), ImmutableMap.of("prometheus.uri", "http://localhost:9090"));
+    Request createRequest = getCreateDataSourceRequest(createDSM);
+    Response response = client().performRequest(createRequest);
+    Assert.assertEquals(201, response.getStatusLine().getStatusCode());
+  }
+
+  @After
+  protected void deleteDataSourceMetadata() throws IOException {
+    Request deleteRequest = getDeleteDataSourceRequest("my_prometheus");
+    Response deleteResponse = client().performRequest(deleteRequest);
+    Assert.assertEquals(204, deleteResponse.getStatusLine().getStatusCode());
+  }
+
   @Test
   @SneakyThrows
   public void testSourceMetricCommand() {

integ-test/src/test/java/org/opensearch/sql/ppl/ShowDataSourcesCommandIT.java

@@ -7,21 +7,27 @@
 
 package org.opensearch.sql.ppl;
 
+import static org.opensearch.sql.legacy.TestUtils.getResponseBody;
 import static org.opensearch.sql.util.MatcherUtils.columnName;
 import static org.opensearch.sql.util.MatcherUtils.rows;
 import static org.opensearch.sql.util.MatcherUtils.verifyColumn;
 import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;
 
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import java.io.IOException;
 import org.json.JSONObject;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.jupiter.api.Test;
+import org.opensearch.client.Request;
+import org.opensearch.client.Response;
+import org.opensearch.sql.datasource.model.DataSourceMetadata;
+import org.opensearch.sql.datasource.model.DataSourceType;
 
 public class ShowDataSourcesCommandIT extends PPLIntegTestCase {
-  @Override
-  protected void init() throws Exception {
-    loadIndex(Index.DATASOURCES);
-  }
 
   /**
    * Integ tests are dependent on self generated metrics in prometheus instance.
@@ -35,6 +41,23 @@ protected static void metricGenerationWait() throws InterruptedException {
     Thread.sleep(10000);
   }
 
+  @Override
+  protected void init() throws InterruptedException, IOException {
+    DataSourceMetadata createDSM =
+        new DataSourceMetadata("my_prometheus", DataSourceType.PROMETHEUS,
+            ImmutableList.of(), ImmutableMap.of("prometheus.uri", "http://localhost:9090"));
+    Request createRequest = getCreateDataSourceRequest(createDSM);
+    Response response = client().performRequest(createRequest);
+    Assert.assertEquals(201, response.getStatusLine().getStatusCode());
+  }
+
+  @After
+  protected void deleteDataSourceMetadata() throws IOException {
+    Request deleteRequest = getDeleteDataSourceRequest("my_prometheus");
+    Response deleteResponse = client().performRequest(deleteRequest);
+    Assert.assertEquals(204, deleteResponse.getStatusLine().getStatusCode());
+  }
+
   @Test
   public void testShowDataSourcesCommands() throws IOException {
     JSONObject result = executeQuery("show datasources");