diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexScan.java index e9746e1fae..80e19cb76d 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexScan.java @@ -109,7 +109,7 @@ private void fetchNextBatch() { @Override public void close() { super.close(); - + client.cleanup(request); } diff --git a/prometheus/build.gradle b/prometheus/build.gradle new file mode 100644 index 0000000000..4f4a7978db --- /dev/null +++ b/prometheus/build.gradle @@ -0,0 +1,101 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { + id 'java-library' + id "io.freefair.lombok" + id 'jacoco' +} + +dependencies { + api project(':core') + api group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}" + implementation "io.github.resilience4j:resilience4j-retry:1.5.0" + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: "${jackson_version}" + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${jackson_version}" + implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: "${jackson_version}" + compileOnly group: 'org.opensearch.client', name: 'opensearch-rest-high-level-client', version: "${opensearch_version}" + api group: 'com.squareup.okhttp3', name: 'okhttp', version: '4.9.3' + implementation group: 'org.json', name: 'json', version: '20180813' + + testImplementation('org.junit.jupiter:junit-jupiter:5.6.2') + testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' + testImplementation group: 'org.mockito', name: 'mockito-core', version: '3.12.4' + testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: '3.12.4' + testImplementation group: 'org.opensearch.client', name: 'opensearch-rest-high-level-client', version: "${opensearch_version}" + testImplementation group: 'org.opensearch.test', name: 'framework', version: "${opensearch_version}" + testImplementation group: 'com.squareup.okhttp3', name: 'mockwebserver', version: '4.9.3' +} + +test { + useJUnitPlatform() + testLogging { + events "passed", "skipped", "failed" + exceptionFormat "full" + } +} + +configurations.all { + resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib:1.6.0" + resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-common:1.6.0" +} + +jacocoTestReport { + reports { + html.enabled true + xml.enabled true + } + afterEvaluate { + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it) + })) + } +} +test.finalizedBy(project.tasks.jacocoTestReport) + +jacocoTestCoverageVerification { + violationRules { + rule { + element = 'CLASS' + excludes = [ + 'org.opensearch.sql.prometheus.data.constants.*' + ] + limit { + counter = 'LINE' + minimum = 1.0 + } + limit { + counter = 'BRANCH' + minimum = 1.0 + } + } + } + afterEvaluate { + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it) + })) + } +} +check.dependsOn jacocoTestCoverageVerification +jacocoTestCoverageVerification.dependsOn jacocoTestReport diff --git a/prometheus/lombok.config b/prometheus/lombok.config new file mode 100644 index 0000000000..aac13295bd --- /dev/null +++ b/prometheus/lombok.config @@ -0,0 +1,3 @@ +# This file is generated by the 'io.freefair.lombok' Gradle plugin +config.stopBubbling = true +lombok.addLombokGeneratedAnnotation = true \ No newline at end of file diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClient.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClient.java new file mode 100644 index 0000000000..bba9a4f1d8 --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClient.java @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.prometheus.client; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.io.IOException; +import java.util.List; +import org.json.JSONObject; + +public interface PrometheusClient { + + JSONObject queryRange(String query, Long start, Long end, String step) throws IOException; + + String[] getLabels(String metricName) throws IOException; + + void schedule(Runnable task); +} \ No newline at end of file diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClientImpl.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClientImpl.java new file mode 100644 index 0000000000..4e63ba9737 --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClientImpl.java @@ -0,0 +1,98 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.prometheus.client; + +import java.io.IOException; +import java.net.URI; +import java.util.Objects; +import okhttp3.HttpUrl; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.json.JSONArray; +import org.json.JSONObject; + +public class PrometheusClientImpl implements PrometheusClient { + + private static final Logger logger = LogManager.getLogger(PrometheusClientImpl.class); + + private final OkHttpClient okHttpClient; + + private final URI uri; + + public PrometheusClientImpl(OkHttpClient okHttpClient, URI uri) { + this.okHttpClient = okHttpClient; + this.uri = uri; + } + + + @Override + public JSONObject queryRange(String query, Long start, Long end, String step) throws IOException { + HttpUrl httpUrl = new HttpUrl.Builder() + .scheme(uri.getScheme()) + .host(uri.getHost()) + .port(uri.getPort()) + .addPathSegment("api") + .addPathSegment("v1") + .addPathSegment("query_range") + .addQueryParameter("query", query) + .addQueryParameter("start", Long.toString(start)) + .addQueryParameter("end", Long.toString(end)) + .addQueryParameter("step", step) + .build(); + logger.debug("queryUrl: " + httpUrl); + Request request = new Request.Builder() + .url(httpUrl) + .build(); + Response response = this.okHttpClient.newCall(request).execute(); + JSONObject jsonObject = readResponse(response); + return jsonObject.getJSONObject("data"); + } + + @Override + public String[] getLabels(String metricName) throws IOException { + String queryUrl = String.format("%sapi/v1/labels?match[]=%s", uri.toString(), metricName); + logger.debug("queryUrl: " + queryUrl); + Request request = new Request.Builder() + .url(queryUrl) + .build(); + Response response = this.okHttpClient.newCall(request).execute(); + JSONObject jsonObject = readResponse(response); + return toStringArray(jsonObject.getJSONArray("data")); + } + + @Override + public void schedule(Runnable task) { + task.run(); + } + + private String[] toStringArray(JSONArray array) { + String[] arr = new String[array.length()]; + for (int i = 0; i < arr.length; i++) { + arr[i] = array.optString(i); + } + return arr; + } + + + private JSONObject readResponse(Response response) throws IOException { + if (response.isSuccessful()) { + JSONObject jsonObject = new JSONObject(Objects.requireNonNull(response.body()).string()); + if ("success".equals(jsonObject.getString("status"))) { + return jsonObject; + } else { + throw new RuntimeException(jsonObject.getString("error")); + } + } else { + throw new RuntimeException( + String.format("Request to Prometheus is Unsuccessful with : %s", response.message())); + } + } + + +} diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/data/constants/PrometheusFieldConstants.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/data/constants/PrometheusFieldConstants.java new file mode 100644 index 0000000000..98d01789e2 --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/data/constants/PrometheusFieldConstants.java @@ -0,0 +1,7 @@ +package org.opensearch.sql.prometheus.data.constants; + +public class PrometheusFieldConstants { + public static final String TIMESTAMP = "@timestamp"; + public static final String VALUE = "@value"; + public static final String METRIC = "metric"; +} diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/request/PrometheusDescribeMetricRequest.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/request/PrometheusDescribeMetricRequest.java new file mode 100644 index 0000000000..753692d1dc --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/request/PrometheusDescribeMetricRequest.java @@ -0,0 +1,75 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.prometheus.request; + +import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.METRIC; +import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.TIMESTAMP; +import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.VALUE; + +import java.io.IOException; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.HashMap; +import java.util.Map; +import lombok.ToString; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.prometheus.client.PrometheusClient; + +/** + * Describe Metric meta data request. + */ +@ToString(onlyExplicitlyIncluded = true) +public class PrometheusDescribeMetricRequest { + + private final PrometheusClient prometheusClient; + + @ToString.Include + private final String metricName; + + private static final Logger LOG = LogManager.getLogger(); + + + public PrometheusDescribeMetricRequest(PrometheusClient prometheusClient, + String metricName) { + this.prometheusClient = prometheusClient; + this.metricName = metricName; + } + + /** + * Get the mapping of field and type. + * + * @return mapping of field and type. + */ + public Map getFieldTypes() { + Map fieldTypes = new HashMap<>(); + + String[] labels = AccessController.doPrivileged((PrivilegedAction) () -> { + if (metricName != null) { + try { + return prometheusClient.getLabels(metricName); + } catch (IOException e) { + LOG.error("Error connecting prometheus., {}", e.getMessage()); + throw new RuntimeException("Error connecting prometheus. " + e.getMessage()); + } + } + return null; + }); + if (labels != null) { + for (String label : labels) { + fieldTypes.put(label, ExprCoreType.STRING); + } + } + fieldTypes.put(VALUE, ExprCoreType.DOUBLE); + fieldTypes.put(TIMESTAMP, ExprCoreType.TIMESTAMP); + fieldTypes.put(METRIC, ExprCoreType.STRING); + return fieldTypes; + } + +} diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/request/PrometheusQueryRequest.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/request/PrometheusQueryRequest.java new file mode 100644 index 0000000000..2c49d01d18 --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/request/PrometheusQueryRequest.java @@ -0,0 +1,42 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.prometheus.request; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.opensearch.common.unit.TimeValue; + +/** + * Prometheus metric query request. + */ +@EqualsAndHashCode +@Getter +@ToString +public class PrometheusQueryRequest { + + public static final TimeValue DEFAULT_QUERY_TIMEOUT = TimeValue.timeValueMinutes(1L); + + private final StringBuilder promQl; + + @Setter + private Long startTime; + + @Setter + private Long endTime; + + @Setter + private String step; + + /** + * Constructor of PrometheusQueryRequest. + */ + public PrometheusQueryRequest() { + this.promQl = new StringBuilder(); + } +} diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/response/PrometheusResponse.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/response/PrometheusResponse.java new file mode 100644 index 0000000000..8f50c7f3b0 --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/response/PrometheusResponse.java @@ -0,0 +1,52 @@ +package org.opensearch.sql.prometheus.response; + +import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.METRIC; +import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.TIMESTAMP; +import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.VALUE; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import lombok.NonNull; +import org.json.JSONArray; +import org.json.JSONObject; +import org.opensearch.sql.data.model.ExprDoubleValue; +import org.opensearch.sql.data.model.ExprStringValue; +import org.opensearch.sql.data.model.ExprTimestampValue; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprValue; + +public class PrometheusResponse implements Iterable { + + private final JSONObject responseObject; + + public PrometheusResponse(JSONObject responseObject) { + this.responseObject = responseObject; + } + + @NonNull + @Override + public Iterator iterator() { + List result = new ArrayList<>(); + if ("matrix".equals(responseObject.getString("resultType"))) { + JSONArray itemArray = responseObject.getJSONArray("result"); + for (int i = 0; i < itemArray.length(); i++) { + JSONObject item = itemArray.getJSONObject(i); + JSONObject metric = item.getJSONObject("metric"); + JSONArray values = item.getJSONArray("values"); + for (int j = 0; j < values.length(); j++) { + LinkedHashMap linkedHashMap = new LinkedHashMap<>(); + JSONArray val = values.getJSONArray(j); + linkedHashMap.put(TIMESTAMP, + new ExprTimestampValue(Instant.ofEpochMilli((long) (val.getDouble(0) * 1000)))); + linkedHashMap.put(VALUE, new ExprDoubleValue(val.getDouble(1))); + linkedHashMap.put(METRIC, new ExprStringValue(metric.toString())); + result.add(new ExprTupleValue(linkedHashMap)); + } + } + } + return result.iterator(); + } +} diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScan.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScan.java new file mode 100644 index 0000000000..718ed344f3 --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScan.java @@ -0,0 +1,78 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.prometheus.storage; + +import java.io.IOException; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.Iterator; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.json.JSONObject; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.prometheus.client.PrometheusClient; +import org.opensearch.sql.prometheus.request.PrometheusQueryRequest; +import org.opensearch.sql.prometheus.response.PrometheusResponse; +import org.opensearch.sql.storage.TableScanOperator; + +/** + * OpenSearch index scan operator. + */ +@EqualsAndHashCode(onlyExplicitlyIncluded = true, callSuper = false) +@ToString(onlyExplicitlyIncluded = true) +public class PrometheusMetricScan extends TableScanOperator { + + private final PrometheusClient prometheusClient; + + @EqualsAndHashCode.Include + @Getter + @ToString.Include + private final PrometheusQueryRequest request; + + private Iterator iterator; + + private static final Logger LOG = LogManager.getLogger(); + + public PrometheusMetricScan(PrometheusClient prometheusClient) { + this.prometheusClient = prometheusClient; + this.request = new PrometheusQueryRequest(); + } + + @Override + public void open() { + super.open(); + this.iterator = AccessController.doPrivileged((PrivilegedAction>) () -> { + try { + JSONObject responseObject = prometheusClient.queryRange( + request.getPromQl().toString(), + request.getStartTime(), request.getEndTime(), request.getStep()); + return new PrometheusResponse(responseObject).iterator(); + } catch (IOException e) { + LOG.error(e.getMessage()); + throw new RuntimeException("Error fetching data from prometheus server. " + e.getMessage()); + } + }); + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public ExprValue next() { + return iterator.next(); + } + + @Override + public String explain() { + return getRequest().toString(); + } +} diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/client/PrometheusClientImplTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/client/PrometheusClientImplTest.java new file mode 100644 index 0000000000..e761a3c911 --- /dev/null +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/client/PrometheusClientImplTest.java @@ -0,0 +1,141 @@ +package org.opensearch.sql.prometheus.client; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.opensearch.sql.prometheus.constants.TestConstants.ENDTIME; +import static org.opensearch.sql.prometheus.constants.TestConstants.METRIC_NAME; +import static org.opensearch.sql.prometheus.constants.TestConstants.QUERY; +import static org.opensearch.sql.prometheus.constants.TestConstants.STARTTIME; +import static org.opensearch.sql.prometheus.constants.TestConstants.STEP; +import static org.opensearch.sql.prometheus.utils.TestUtils.getJson; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicBoolean; +import lombok.SneakyThrows; +import okhttp3.HttpUrl; +import okhttp3.OkHttpClient; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import okhttp3.mockwebserver.RecordedRequest; +import org.apache.http.HttpStatus; +import org.json.JSONObject; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class PrometheusClientImplTest { + + private MockWebServer mockWebServer; + private PrometheusClient prometheusClient; + + + @BeforeEach + void setUp() throws IOException { + this.mockWebServer = new MockWebServer(); + this.mockWebServer.start(); + this.prometheusClient = + new PrometheusClientImpl(new OkHttpClient(), mockWebServer.url("/").uri()); + } + + + @Test + @SneakyThrows + void testQueryRange() { + MockResponse mockResponse = new MockResponse() + .addHeader("Content-Type", "application/json; charset=utf-8") + .setBody(getJson("query_range_response.json")); + mockWebServer.enqueue(mockResponse); + JSONObject jsonObject = prometheusClient.queryRange(QUERY, STARTTIME, ENDTIME, STEP); + assertTrue(new JSONObject(getJson("query_range_result.json")).similar(jsonObject)); + RecordedRequest recordedRequest = mockWebServer.takeRequest(); + verifyQueryRangeCall(recordedRequest); + } + + @Test + @SneakyThrows + void testQueryRangeWith2xxStatusAndError() { + MockResponse mockResponse = new MockResponse() + .addHeader("Content-Type", "application/json; charset=utf-8") + .setBody(getJson("error_response.json")); + mockWebServer.enqueue(mockResponse); + RuntimeException runtimeException + = assertThrows(RuntimeException.class, + () -> prometheusClient.queryRange(QUERY, STARTTIME, ENDTIME, STEP)); + assertEquals("Error", runtimeException.getMessage()); + RecordedRequest recordedRequest = mockWebServer.takeRequest(); + verifyQueryRangeCall(recordedRequest); + } + + @Test + @SneakyThrows + void testQueryRangeWithNon2xxError() { + MockResponse mockResponse = new MockResponse() + .addHeader("Content-Type", "application/json; charset=utf-8") + .setResponseCode(HttpStatus.SC_BAD_REQUEST); + mockWebServer.enqueue(mockResponse); + RuntimeException runtimeException + = assertThrows(RuntimeException.class, + () -> prometheusClient.queryRange(QUERY, STARTTIME, ENDTIME, STEP)); + assertTrue( + runtimeException.getMessage().contains("Request to Prometheus is Unsuccessful with :")); + RecordedRequest recordedRequest = mockWebServer.takeRequest(); + verifyQueryRangeCall(recordedRequest); + } + + @Test + @SneakyThrows + void testGetLabel() { + MockResponse mockResponse = new MockResponse() + .addHeader("Content-Type", "application/json; charset=utf-8") + .setBody(getJson("get_labels_response.json")); + mockWebServer.enqueue(mockResponse); + String[] response = prometheusClient.getLabels(METRIC_NAME); + assertArrayEquals(new String[] { + "__name__", + "call", + "code"}, response); + RecordedRequest recordedRequest = mockWebServer.takeRequest(); + verifyGetLabelsCall(recordedRequest); + } + + @Test + void schedule() { + AtomicBoolean isRun = new AtomicBoolean(false); + prometheusClient.schedule( + () -> { + isRun.set(true); + }); + assertTrue(isRun.get()); + } + + @AfterEach + void tearDown() throws IOException { + mockWebServer.shutdown(); + } + + private void verifyQueryRangeCall(RecordedRequest recordedRequest) { + HttpUrl httpUrl = recordedRequest.getRequestUrl(); + assertEquals("GET", recordedRequest.getMethod()); + assertNotNull(httpUrl); + assertEquals("/api/v1/query_range", httpUrl.encodedPath()); + assertEquals(QUERY, httpUrl.queryParameter("query")); + assertEquals(STARTTIME.toString(), httpUrl.queryParameter("start")); + assertEquals(ENDTIME.toString(), httpUrl.queryParameter("end")); + assertEquals(STEP, httpUrl.queryParameter("step")); + } + + private void verifyGetLabelsCall(RecordedRequest recordedRequest) { + HttpUrl httpUrl = recordedRequest.getRequestUrl(); + assertEquals("GET", recordedRequest.getMethod()); + assertNotNull(httpUrl); + assertEquals("/api/v1/labels", httpUrl.encodedPath()); + assertEquals(METRIC_NAME, httpUrl.queryParameter("match[]")); + } + +} diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/constants/TestConstants.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/constants/TestConstants.java new file mode 100644 index 0000000000..7c0fb78b30 --- /dev/null +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/constants/TestConstants.java @@ -0,0 +1,9 @@ +package org.opensearch.sql.prometheus.constants; + +public class TestConstants { + public static final String QUERY = "test_query"; + public static final Long STARTTIME = 1664767694133L; + public static final Long ENDTIME = 1664771294133L; + public static final String STEP = "14"; + public static final String METRIC_NAME = "http_requests_total"; +} diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/request/PrometheusDescribeMetricRequestTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/request/PrometheusDescribeMetricRequestTest.java new file mode 100644 index 0000000000..527968fd60 --- /dev/null +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/request/PrometheusDescribeMetricRequestTest.java @@ -0,0 +1,92 @@ +package org.opensearch.sql.prometheus.request; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.prometheus.constants.TestConstants.METRIC_NAME; +import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.METRIC; +import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.TIMESTAMP; +import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.VALUE; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import lombok.SneakyThrows; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.prometheus.client.PrometheusClient; + +@ExtendWith(MockitoExtension.class) +public class PrometheusDescribeMetricRequestTest { + + @Mock + private PrometheusClient prometheusClient; + + @Test + @SneakyThrows + void testGetFieldTypes() { + when(prometheusClient.getLabels(METRIC_NAME)).thenReturn(new String[] { + "call", + "code" + }); + Map expected = new HashMap<>() {{ + put("call", ExprCoreType.STRING); + put("code", ExprCoreType.STRING); + put(VALUE, ExprCoreType.DOUBLE); + put(TIMESTAMP, ExprCoreType.TIMESTAMP); + put(METRIC, ExprCoreType.STRING); + }}; + PrometheusDescribeMetricRequest prometheusDescribeMetricRequest + = new PrometheusDescribeMetricRequest(prometheusClient, METRIC_NAME); + assertEquals(expected, prometheusDescribeMetricRequest.getFieldTypes()); + verify(prometheusClient, times(1)).getLabels(METRIC_NAME); + } + + + @Test + @SneakyThrows + void testGetFieldTypesWithEmptyMetricName() { + Map expected = new HashMap<>() {{ + put(VALUE, ExprCoreType.DOUBLE); + put(TIMESTAMP, ExprCoreType.TIMESTAMP); + put(METRIC, ExprCoreType.STRING); + }}; + PrometheusDescribeMetricRequest prometheusDescribeMetricRequest + = new PrometheusDescribeMetricRequest(prometheusClient, null); + assertEquals(expected, prometheusDescribeMetricRequest.getFieldTypes()); + verifyNoInteractions(prometheusClient); + } + + + @Test + @SneakyThrows + void testGetFieldTypesWhenException() { + when(prometheusClient.getLabels(METRIC_NAME)).thenThrow(new RuntimeException("ERROR Message")); + PrometheusDescribeMetricRequest prometheusDescribeMetricRequest + = new PrometheusDescribeMetricRequest(prometheusClient, METRIC_NAME); + RuntimeException exception = assertThrows(RuntimeException.class, + prometheusDescribeMetricRequest::getFieldTypes); + verify(prometheusClient, times(1)).getLabels(METRIC_NAME); + assertEquals("ERROR Message", exception.getMessage()); + } + + @Test + @SneakyThrows + void testGetFieldTypesWhenIOException() { + when(prometheusClient.getLabels(METRIC_NAME)).thenThrow(new IOException("ERROR Message")); + PrometheusDescribeMetricRequest prometheusDescribeMetricRequest + = new PrometheusDescribeMetricRequest(prometheusClient, METRIC_NAME); + RuntimeException exception = assertThrows(RuntimeException.class, + prometheusDescribeMetricRequest::getFieldTypes); + assertEquals("Error connecting prometheus. ERROR Message", exception.getMessage()); + verify(prometheusClient, times(1)).getLabels(METRIC_NAME); + } + +} diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScanTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScanTest.java new file mode 100644 index 0000000000..d1b2c586b5 --- /dev/null +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScanTest.java @@ -0,0 +1,129 @@ +package org.opensearch.sql.prometheus.storage; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.prometheus.constants.TestConstants.ENDTIME; +import static org.opensearch.sql.prometheus.constants.TestConstants.QUERY; +import static org.opensearch.sql.prometheus.constants.TestConstants.STARTTIME; +import static org.opensearch.sql.prometheus.constants.TestConstants.STEP; +import static org.opensearch.sql.prometheus.utils.TestUtils.getJson; + +import java.io.IOException; +import java.time.Instant; +import java.util.LinkedHashMap; +import lombok.SneakyThrows; +import org.json.JSONObject; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.data.model.ExprDoubleValue; +import org.opensearch.sql.data.model.ExprStringValue; +import org.opensearch.sql.data.model.ExprTimestampValue; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.prometheus.client.PrometheusClient; + +@ExtendWith(MockitoExtension.class) +public class PrometheusMetricScanTest { + + @Mock + private PrometheusClient prometheusClient; + + @Test + @SneakyThrows + void testQueryResponseIterator() { + PrometheusMetricScan prometheusMetricScan = new PrometheusMetricScan(prometheusClient); + prometheusMetricScan.getRequest().getPromQl().append(QUERY); + prometheusMetricScan.getRequest().setStartTime(STARTTIME); + prometheusMetricScan.getRequest().setEndTime(ENDTIME); + prometheusMetricScan.getRequest().setStep(STEP); + + when(prometheusClient.queryRange(any(), any(), any(), any())) + .thenReturn(new JSONObject(getJson("query_range_result.json"))); + prometheusMetricScan.open(); + Assertions.assertTrue(prometheusMetricScan.hasNext()); + ExprTupleValue firstRow = new ExprTupleValue(new LinkedHashMap<>() {{ + put("@timestamp", new ExprTimestampValue(Instant.ofEpochMilli(1435781430781L))); + put("@value", new ExprDoubleValue(1)); + put("metric", new ExprStringValue( + "{\"instance\":\"localhost:9090\",\"__name__\":\"up\",\"job\":\"prometheus\"}")); + } + }); + assertEquals(firstRow, prometheusMetricScan.next()); + Assertions.assertTrue(prometheusMetricScan.hasNext()); + ExprTupleValue secondRow = new ExprTupleValue(new LinkedHashMap<>() {{ + put("@timestamp", new ExprTimestampValue(Instant.ofEpochMilli(1435781430781L))); + put("@value", new ExprDoubleValue(0)); + put("metric", new ExprStringValue( + "{\"instance\":\"localhost:9091\",\"__name__\":\"up\",\"job\":\"node\"}")); + } + }); + assertEquals(secondRow, prometheusMetricScan.next()); + Assertions.assertFalse(prometheusMetricScan.hasNext()); + } + + @Test + @SneakyThrows + void testEmptyQueryResponseIterator() { + PrometheusMetricScan prometheusMetricScan = new PrometheusMetricScan(prometheusClient); + prometheusMetricScan.getRequest().getPromQl().append(QUERY); + prometheusMetricScan.getRequest().setStartTime(STARTTIME); + prometheusMetricScan.getRequest().setEndTime(ENDTIME); + prometheusMetricScan.getRequest().setStep(STEP); + + when(prometheusClient.queryRange(any(), any(), any(), any())) + .thenReturn(new JSONObject(getJson("empty_query_range_result.json"))); + prometheusMetricScan.open(); + Assertions.assertFalse(prometheusMetricScan.hasNext()); + } + + @Test + @SneakyThrows + void testEmptyQueryWithNoMatrixKeyinResultJson() { + PrometheusMetricScan prometheusMetricScan = new PrometheusMetricScan(prometheusClient); + prometheusMetricScan.getRequest().getPromQl().append(QUERY); + prometheusMetricScan.getRequest().setStartTime(STARTTIME); + prometheusMetricScan.getRequest().setEndTime(ENDTIME); + prometheusMetricScan.getRequest().setStep(STEP); + + when(prometheusClient.queryRange(any(), any(), any(), any())) + .thenReturn(new JSONObject(getJson("no_matrix_query_range_result.json"))); + prometheusMetricScan.open(); + Assertions.assertFalse(prometheusMetricScan.hasNext()); + } + + @Test + @SneakyThrows + void testEmptyQueryWithException() { + PrometheusMetricScan prometheusMetricScan = new PrometheusMetricScan(prometheusClient); + prometheusMetricScan.getRequest().getPromQl().append(QUERY); + prometheusMetricScan.getRequest().setStartTime(STARTTIME); + prometheusMetricScan.getRequest().setEndTime(ENDTIME); + prometheusMetricScan.getRequest().setStep(STEP); + + when(prometheusClient.queryRange(any(), any(), any(), any())) + .thenThrow(new IOException("Error Message")); + RuntimeException runtimeException + = assertThrows(RuntimeException.class, prometheusMetricScan::open); + assertEquals("Error fetching data from prometheus server. Error Message", + runtimeException.getMessage()); + } + + @Test + @SneakyThrows + void testExplain() { + PrometheusMetricScan prometheusMetricScan = new PrometheusMetricScan(prometheusClient); + prometheusMetricScan.getRequest().getPromQl().append(QUERY); + prometheusMetricScan.getRequest().setStartTime(STARTTIME); + prometheusMetricScan.getRequest().setEndTime(ENDTIME); + prometheusMetricScan.getRequest().setStep(STEP); + assertEquals( + "PrometheusQueryRequest(promQl=test_query, startTime=1664767694133, " + + "endTime=1664771294133, step=14)", + prometheusMetricScan.explain()); + } + +} diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/utils/TestUtils.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/utils/TestUtils.java new file mode 100644 index 0000000000..2019b9b7f5 --- /dev/null +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/utils/TestUtils.java @@ -0,0 +1,20 @@ +package org.opensearch.sql.prometheus.utils; + +import java.io.IOException; +import java.util.Objects; + +public class TestUtils { + + /** + * Get Json document from the files in resources folder. + * @param filename filename. + * @return String. + * @throws IOException IOException. + */ + public static String getJson(String filename) throws IOException { + ClassLoader classLoader = TestUtils.class.getClassLoader(); + return new String( + Objects.requireNonNull(classLoader.getResourceAsStream(filename)).readAllBytes()); + } + +} diff --git a/prometheus/src/test/resources/empty_query_range_result.json b/prometheus/src/test/resources/empty_query_range_result.json new file mode 100644 index 0000000000..c245fe6a5d --- /dev/null +++ b/prometheus/src/test/resources/empty_query_range_result.json @@ -0,0 +1,5 @@ +{ + "resultType" : "matrix", + "result" : [ + ] +} \ No newline at end of file diff --git a/prometheus/src/test/resources/error_response.json b/prometheus/src/test/resources/error_response.json new file mode 100644 index 0000000000..07e0208993 --- /dev/null +++ b/prometheus/src/test/resources/error_response.json @@ -0,0 +1,5 @@ +{ + "status" : "error", + "errorType" : "Error", + "error" : "Error" +} \ No newline at end of file diff --git a/prometheus/src/test/resources/get_labels_response.json b/prometheus/src/test/resources/get_labels_response.json new file mode 100644 index 0000000000..48678ea70f --- /dev/null +++ b/prometheus/src/test/resources/get_labels_response.json @@ -0,0 +1,8 @@ +{ + "status": "success", + "data": [ + "__name__", + "call", + "code" + ] +} \ No newline at end of file diff --git a/prometheus/src/test/resources/no_matrix_query_range_result.json b/prometheus/src/test/resources/no_matrix_query_range_result.json new file mode 100644 index 0000000000..c955474b43 --- /dev/null +++ b/prometheus/src/test/resources/no_matrix_query_range_result.json @@ -0,0 +1,25 @@ +{ + "resultType" : "vector", + "result" : [ + { + "metric" : { + "__name__" : "up", + "job" : "prometheus", + "instance" : "localhost:9090" + }, + "values" : [ + [ 1435781430.781, "1" ] + ] + }, + { + "metric" : { + "__name__" : "up", + "job" : "node", + "instance" : "localhost:9091" + }, + "values" : [ + [ 1435781430.781, "0" ] + ] + } + ] +} \ No newline at end of file diff --git a/prometheus/src/test/resources/query_range_response.json b/prometheus/src/test/resources/query_range_response.json new file mode 100644 index 0000000000..57bf06119e --- /dev/null +++ b/prometheus/src/test/resources/query_range_response.json @@ -0,0 +1,28 @@ +{ + "status" : "success", + "data" : { + "resultType" : "matrix", + "result" : [ + { + "metric" : { + "__name__" : "up", + "job" : "prometheus", + "instance" : "localhost:9090" + }, + "values" : [ + [ 1435781430.781, "1" ] + ] + }, + { + "metric" : { + "__name__" : "up", + "job" : "node", + "instance" : "localhost:9091" + }, + "values" : [ + [ 1435781430.781, "0" ] + ] + } + ] + } +} \ No newline at end of file diff --git a/prometheus/src/test/resources/query_range_result.json b/prometheus/src/test/resources/query_range_result.json new file mode 100644 index 0000000000..139b9946a1 --- /dev/null +++ b/prometheus/src/test/resources/query_range_result.json @@ -0,0 +1,25 @@ +{ + "resultType" : "matrix", + "result" : [ + { + "metric" : { + "__name__" : "up", + "job" : "prometheus", + "instance" : "localhost:9090" + }, + "values" : [ + [ 1435781430.781, "1" ] + ] + }, + { + "metric" : { + "__name__" : "up", + "job" : "node", + "instance" : "localhost:9091" + }, + "values" : [ + [ 1435781430.781, "0" ] + ] + } + ] +} \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index 3d4700d8de..2f850f422b 100644 --- a/settings.gradle +++ b/settings.gradle @@ -17,4 +17,5 @@ include 'protocol' include 'doctest' include 'legacy' include 'sql' +include 'prometheus'