added doctest, integ-tests, and unit tests

opensearch-project · Jan 6, 2025 · f84e21d · f84e21d
1 parent 70152eb
commit f84e21d
Show file tree

Hide file tree

Showing 12 changed files with 204 additions and 3 deletions.
diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.expression.json;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.when;
+import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE;
+import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE;
+import static org.opensearch.sql.data.type.ExprCoreType.STRING;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.opensearch.sql.data.model.ExprValue;
+import org.opensearch.sql.data.model.ExprValueUtils;
+import org.opensearch.sql.expression.DSL;
+import org.opensearch.sql.expression.Expression;
+import org.opensearch.sql.expression.FunctionExpression;
+import org.opensearch.sql.expression.env.Environment;
+
+@ExtendWith(MockitoExtension.class)
+public class JsonFunctionsTest {
+
+    private static final ExprValue JsonObject = ExprValueUtils.stringValue("{\"a\":\"1\",\"b\":\"2\"}");
+    private static final ExprValue JsonArray = ExprValueUtils.stringValue("[1, 2, 3, 4]");
+    private static final ExprValue JsonScalarString = ExprValueUtils.stringValue("\"abc\"");
+    private static final ExprValue JsonEmptyString = ExprValueUtils.stringValue("");
+    private static final ExprValue JsonInvalidObject = ExprValueUtils.stringValue("{\"invalid\":\"json\", \"string\"}");
+    private static final ExprValue JsonInvalidScalar = ExprValueUtils.stringValue("abc");
+
+    @Mock private Environment<Expression, ExprValue> env;
+
+    @Test
+    public void json_valid_invalid_json_string() {
+        assertEquals(LITERAL_FALSE, execute(JsonInvalidObject));
+        assertEquals(LITERAL_FALSE, execute(JsonInvalidScalar));
+    }
+
+    @Test
+    public void json_valid_valid_json_string() {
+        assertEquals(LITERAL_TRUE, JsonObject);
+        assertEquals(LITERAL_TRUE, JsonArray);
+        assertEquals(LITERAL_TRUE, JsonScalarString);
+        assertEquals(LITERAL_TRUE, JsonEmptyString);
+    }
+
+    private ExprValue execute(ExprValue jsonString) {
+        final String fieldName = "json_string";
+        FunctionExpression exp = DSL.jsonValid(DSL.literal(jsonString));
+
+        when(DSL.ref(fieldName, STRING).valueOf(env)).thenReturn(jsonString);
+
+        return exp.valueOf(env);
+    }
+}
diff --git a/docs/category.json b/docs/category.json
@@ -34,6 +34,7 @@
     "user/ppl/functions/datetime.rst",
     "user/ppl/functions/expressions.rst",
     "user/ppl/functions/ip.rst",
+    "user/ppl/functions/json.rst",
     "user/ppl/functions/math.rst",
     "user/ppl/functions/relevance.rst",
     "user/ppl/functions/string.rst"

diff --git a/docs/user/dql/metadata.rst b/docs/user/dql/metadata.rst
@@ -35,7 +35,7 @@ Example 1: Show All Indices Information
 SQL query::
 
     os> SHOW TABLES LIKE '%'
-    fetched rows / total rows = 10/10
+    fetched rows / total rows = 11/11
     +----------------+-------------+-----------------+------------+---------+----------+------------+-----------+---------------------------+----------------+
     | TABLE_CAT      | TABLE_SCHEM | TABLE_NAME      | TABLE_TYPE | REMARKS | TYPE_CAT | TYPE_SCHEM | TYPE_NAME | SELF_REFERENCING_COL_NAME | REF_GENERATION |
     |----------------+-------------+-----------------+------------+---------+----------+------------+-----------+---------------------------+----------------|
@@ -44,6 +44,7 @@ SQL query::
     | docTestCluster | null        | accounts        | BASE TABLE | null    | null     | null       | null      | null                      | null           |
     | docTestCluster | null        | apache          | BASE TABLE | null    | null     | null       | null      | null                      | null           |
     | docTestCluster | null        | books           | BASE TABLE | null    | null     | null       | null      | null                      | null           |
+    | docTestCluster | null        | json_test       | BASE TABLE | null    | null     | null       | null      | null                      | null           |
     | docTestCluster | null        | nested          | BASE TABLE | null    | null     | null       | null      | null                      | null           |
     | docTestCluster | null        | nyc_taxi        | BASE TABLE | null    | null     | null       | null      | null                      | null           |
     | docTestCluster | null        | people          | BASE TABLE | null    | null     | null       | null      | null                      | null           |

diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst
@@ -0,0 +1,34 @@
+====================
+IP Address Functions
+====================
+
+.. rubric:: Table of contents
+
+.. contents::
+   :local:
+   :depth: 1
+
+JSON_VALID
+----------
+
+Description
+>>>>>>>>>>>
+
+Usage: `json_valid(json_string)` checks if `json_string` is a valid STRING string.
+
+Argument type: STRING
+
+Return type: BOOLEAN
+
+Example::
+
+    > source=json_test | where json_valid(json_string) | fields test_name, json_string
+    fetched rows / total rows = 4/4
+    +--------------------+--------------------+
+    | test_name          | json_string        |
+    |--------------------|--------------------|
+    | json object        | {"a":"1","b":"2"}  |
+    | json array         | [1, 2, 3, 4]       |
+    | json scalar string | [1, 2, 3, 4]       |
+    | json empty string  | [1, 2, 3, 4]       |
+    +--------------------+--------------------+
diff --git a/doctest/test_data/json_test.json b/doctest/test_data/json_test.json
@@ -0,0 +1,5 @@
+{"test_name":"json object", "json_string":"{\"a\":\"1\",\"b\":\"2\"}"}
+{"test_name":"json array", "json_string":"[1, 2, 3, 4]"}
+{"test_name":"json scalar string", "json_string":"\"abc\""}
+{"test_name":"json empty string","json_string":""}
+{"test_name":"json invalid object", "json_string":"{\"invalid\":\"json\", \"string\"}"}
diff --git a/doctest/test_docs.py b/doctest/test_docs.py
@@ -30,6 +30,7 @@
 NESTED = "nested"
 DATASOURCES = ".ql-datasources"
 WEBLOGS = "weblogs"
+JSON_TEST = "json_test"
 
 class DocTestConnection(OpenSearchConnection):
 
@@ -123,6 +124,7 @@ def set_up_test_indices(test):
     load_file("nested_objects.json", index_name=NESTED)
     load_file("datasources.json", index_name=DATASOURCES)
     load_file("weblogs.json", index_name=WEBLOGS)
+    load_file("json_test.json", index_name=JSON_TEST)
 
 
 def load_file(filename, index_name):
@@ -151,7 +153,7 @@ def set_up(test):
 
 def tear_down(test):
     # drop leftover tables after each test
-    test_data_client.indices.delete(index=[ACCOUNTS, EMPLOYEES, PEOPLE, ACCOUNT2, NYC_TAXI, BOOKS, APACHE, WILDCARD, NESTED, WEBLOGS], ignore_unavailable=True)
+    test_data_client.indices.delete(index=[ACCOUNTS, EMPLOYEES, PEOPLE, ACCOUNT2, NYC_TAXI, BOOKS, APACHE, WILDCARD, NESTED, WEBLOGS, JSON_TEST], ignore_unavailable=True)
 
 
 docsuite = partial(doctest.DocFileSuite,

diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java
@@ -22,6 +22,7 @@
 import static org.opensearch.sql.legacy.TestUtils.getGameOfThronesIndexMapping;
 import static org.opensearch.sql.legacy.TestUtils.getGeopointIndexMapping;
 import static org.opensearch.sql.legacy.TestUtils.getJoinTypeIndexMapping;
+import static org.opensearch.sql.legacy.TestUtils.getJsonTestIndexMapping;
 import static org.opensearch.sql.legacy.TestUtils.getLocationIndexMapping;
 import static org.opensearch.sql.legacy.TestUtils.getMappingFile;
 import static org.opensearch.sql.legacy.TestUtils.getNestedSimpleIndexMapping;
@@ -745,7 +746,12 @@ public enum Index {
         TestsConstants.TEST_INDEX_GEOPOINT,
         "dates",
         getGeopointIndexMapping(),
-        "src/test/resources/geopoints.json");
+        "src/test/resources/geopoints.json"),
+    JSON_TEST(
+        TestsConstants.TEST_INDEX_JSON_TEST,
+        "json",
+        getJsonTestIndexMapping(),
+        "src/test/resources/json_test.json");
 
     private final String name;
     private final String type;

diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java
@@ -250,6 +250,11 @@ public static String getGeopointIndexMapping() {
     return getMappingFile(mappingFile);
   }
 
+  public static String getJsonTestIndexMapping() {
+    String mappingFile = "json_test_index_mapping.json";
+    return getMappingFile(mappingFile);
+  }
+
   public static void loadBulk(Client client, String jsonPath, String defaultIndex)
       throws Exception {
     System.out.println(String.format("Loading file %s into opensearch cluster", jsonPath));

diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java
@@ -58,6 +58,7 @@ public class TestsConstants {
   public static final String TEST_INDEX_MULTI_NESTED_TYPE = TEST_INDEX + "_multi_nested";
   public static final String TEST_INDEX_NESTED_WITH_NULLS = TEST_INDEX + "_nested_with_nulls";
   public static final String TEST_INDEX_GEOPOINT = TEST_INDEX + "_geopoint";
+  public static final String TEST_INDEX_JSON_TEST = TEST_INDEX + "_json_test";
   public static final String DATASOURCES = ".ql-datasources";
 
   public static final String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'";

diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.ppl;
+
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import org.json.JSONObject;
+
+import javax.json.Json;
+
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_JSON_TEST;
+import static org.opensearch.sql.util.MatcherUtils.rows;
+import static org.opensearch.sql.util.MatcherUtils.schema;
+import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;
+import static org.opensearch.sql.util.MatcherUtils.verifySchema;
+
+public class JsonFunctionIT extends PPLIntegTestCase {
+    @Override
+    public void init() throws IOException {
+        loadIndex(Index.JSON_TEST);
+    }
+
+    @Test
+    public void test_json_valid() throws IOException {
+        JSONObject result;
+
+        result =
+            executeQuery(
+                String.format(
+                    "source=%s | where json_valid(json_string) | fields test_name",
+                    TEST_INDEX_JSON_TEST
+                )
+            );
+        verifySchema(result, schema("test_name", null,  "string"));
+        verifyDataRows(
+                result,
+                rows("json object"),
+                rows("json array"),
+                rows("json scalar string"),
+                rows("json empty string")
+        );
+    }
+
+    @Test
+    public void test_not_json_valid() throws IOException {
+        JSONObject result;
+
+        result =
+                executeQuery(
+                        String.format(
+                                "source=%s | where not json_valid(json_string) | fields test_name",
+                                TEST_INDEX_JSON_TEST
+                        )
+                );
+        verifySchema(result, schema("test_name", null,  "string"));
+        verifyDataRows(
+                result,
+                rows("json invalid object")
+        );
+    }
+}
diff --git a/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json b/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json
@@ -0,0 +1,12 @@
+{
+  "mappings": {
+    "properties": {
+      "test_name": {
+        "type": "text"
+      },
+      "json_string": {
+        "type": "text"
+      }
+    }
+  }
+}
diff --git a/integ-test/src/test/resources/json_test.json b/integ-test/src/test/resources/json_test.json
@@ -0,0 +1,10 @@
+{"index":{"_id":"1"}}
+{"test_name":"json object", "json_string":"{\"a\":\"1\",\"b\":\"2\"}"}
+{"index":{"_id":"2"}}
+{"test_name":"json array", "json_string":"[1, 2, 3, 4]"}
+{"index":{"_id":"3"}}
+{"test_name":"json scalar string", "json_string":"\"abc\""}
+{"index":{"_id":"4"}}
+{"test_name":"json empty string","json_string":""}
+{"index":{"_id":"5"}}
+{"test_name":"json invalid object", "json_string":"{\"invalid\":\"json\", \"string\"}"}