From f84e21dc63a807e10e3708e1ceb4e4281b45a49a Mon Sep 17 00:00:00 2001 From: Kenrick Yap <14yapkc1@gmail.com> Date: Mon, 6 Jan 2025 15:14:25 -0800 Subject: [PATCH] added doctest, integ-tests, and unit tests --- .../expression/json/JsonFunctionsTest.java | 59 +++++++++++++++++ docs/category.json | 1 + docs/user/dql/metadata.rst | 3 +- docs/user/ppl/functions/json.rst | 34 ++++++++++ doctest/test_data/json_test.json | 5 ++ doctest/test_docs.py | 4 +- .../sql/legacy/SQLIntegTestCase.java | 8 ++- .../org/opensearch/sql/legacy/TestUtils.java | 5 ++ .../opensearch/sql/legacy/TestsConstants.java | 1 + .../opensearch/sql/ppl/JsonFunctionIT.java | 65 +++++++++++++++++++ .../json_test_index_mappping.json | 12 ++++ integ-test/src/test/resources/json_test.json | 10 +++ 12 files changed, 204 insertions(+), 3 deletions(-) create mode 100644 core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java create mode 100644 docs/user/ppl/functions/json.rst create mode 100644 doctest/test_data/json_test.json create mode 100644 integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java create mode 100644 integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json create mode 100644 integ-test/src/test/resources/json_test.json diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java new file mode 100644 index 0000000000..ee817dc71a --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -0,0 +1,59 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.json; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.env.Environment; + +@ExtendWith(MockitoExtension.class) +public class JsonFunctionsTest { + + private static final ExprValue JsonObject = ExprValueUtils.stringValue("{\"a\":\"1\",\"b\":\"2\"}"); + private static final ExprValue JsonArray = ExprValueUtils.stringValue("[1, 2, 3, 4]"); + private static final ExprValue JsonScalarString = ExprValueUtils.stringValue("\"abc\""); + private static final ExprValue JsonEmptyString = ExprValueUtils.stringValue(""); + private static final ExprValue JsonInvalidObject = ExprValueUtils.stringValue("{\"invalid\":\"json\", \"string\"}"); + private static final ExprValue JsonInvalidScalar = ExprValueUtils.stringValue("abc"); + + @Mock private Environment env; + + @Test + public void json_valid_invalid_json_string() { + assertEquals(LITERAL_FALSE, execute(JsonInvalidObject)); + assertEquals(LITERAL_FALSE, execute(JsonInvalidScalar)); + } + + @Test + public void json_valid_valid_json_string() { + assertEquals(LITERAL_TRUE, JsonObject); + assertEquals(LITERAL_TRUE, JsonArray); + assertEquals(LITERAL_TRUE, JsonScalarString); + assertEquals(LITERAL_TRUE, JsonEmptyString); + } + + private ExprValue execute(ExprValue jsonString) { + final String fieldName = "json_string"; + FunctionExpression exp = DSL.jsonValid(DSL.literal(jsonString)); + + when(DSL.ref(fieldName, STRING).valueOf(env)).thenReturn(jsonString); + + return exp.valueOf(env); + } +} diff --git a/docs/category.json b/docs/category.json index 32f56cfb46..efbb57d6e6 100644 --- a/docs/category.json +++ b/docs/category.json @@ -34,6 +34,7 @@ "user/ppl/functions/datetime.rst", "user/ppl/functions/expressions.rst", "user/ppl/functions/ip.rst", + "user/ppl/functions/json.rst", "user/ppl/functions/math.rst", "user/ppl/functions/relevance.rst", "user/ppl/functions/string.rst" diff --git a/docs/user/dql/metadata.rst b/docs/user/dql/metadata.rst index aba4eb0c75..b059c0cded 100644 --- a/docs/user/dql/metadata.rst +++ b/docs/user/dql/metadata.rst @@ -35,7 +35,7 @@ Example 1: Show All Indices Information SQL query:: os> SHOW TABLES LIKE '%' - fetched rows / total rows = 10/10 + fetched rows / total rows = 11/11 +----------------+-------------+-----------------+------------+---------+----------+------------+-----------+---------------------------+----------------+ | TABLE_CAT | TABLE_SCHEM | TABLE_NAME | TABLE_TYPE | REMARKS | TYPE_CAT | TYPE_SCHEM | TYPE_NAME | SELF_REFERENCING_COL_NAME | REF_GENERATION | |----------------+-------------+-----------------+------------+---------+----------+------------+-----------+---------------------------+----------------| @@ -44,6 +44,7 @@ SQL query:: | docTestCluster | null | accounts | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | apache | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | books | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | json_test | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | nested | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | nyc_taxi | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | people | BASE TABLE | null | null | null | null | null | null | diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst new file mode 100644 index 0000000000..8f986ace6b --- /dev/null +++ b/docs/user/ppl/functions/json.rst @@ -0,0 +1,34 @@ +==================== +IP Address Functions +==================== + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 1 + +JSON_VALID +---------- + +Description +>>>>>>>>>>> + +Usage: `json_valid(json_string)` checks if `json_string` is a valid STRING string. + +Argument type: STRING + +Return type: BOOLEAN + +Example:: + + > source=json_test | where json_valid(json_string) | fields test_name, json_string + fetched rows / total rows = 4/4 + +--------------------+--------------------+ + | test_name | json_string | + |--------------------|--------------------| + | json object | {"a":"1","b":"2"} | + | json array | [1, 2, 3, 4] | + | json scalar string | [1, 2, 3, 4] | + | json empty string | [1, 2, 3, 4] | + +--------------------+--------------------+ diff --git a/doctest/test_data/json_test.json b/doctest/test_data/json_test.json new file mode 100644 index 0000000000..2da491675e --- /dev/null +++ b/doctest/test_data/json_test.json @@ -0,0 +1,5 @@ +{"test_name":"json object", "json_string":"{\"a\":\"1\",\"b\":\"2\"}"} +{"test_name":"json array", "json_string":"[1, 2, 3, 4]"} +{"test_name":"json scalar string", "json_string":"\"abc\""} +{"test_name":"json empty string","json_string":""} +{"test_name":"json invalid object", "json_string":"{\"invalid\":\"json\", \"string\"}"} diff --git a/doctest/test_docs.py b/doctest/test_docs.py index 1d46766c6d..906bbd65b5 100644 --- a/doctest/test_docs.py +++ b/doctest/test_docs.py @@ -30,6 +30,7 @@ NESTED = "nested" DATASOURCES = ".ql-datasources" WEBLOGS = "weblogs" +JSON_TEST = "json_test" class DocTestConnection(OpenSearchConnection): @@ -123,6 +124,7 @@ def set_up_test_indices(test): load_file("nested_objects.json", index_name=NESTED) load_file("datasources.json", index_name=DATASOURCES) load_file("weblogs.json", index_name=WEBLOGS) + load_file("json_test.json", index_name=JSON_TEST) def load_file(filename, index_name): @@ -151,7 +153,7 @@ def set_up(test): def tear_down(test): # drop leftover tables after each test - test_data_client.indices.delete(index=[ACCOUNTS, EMPLOYEES, PEOPLE, ACCOUNT2, NYC_TAXI, BOOKS, APACHE, WILDCARD, NESTED, WEBLOGS], ignore_unavailable=True) + test_data_client.indices.delete(index=[ACCOUNTS, EMPLOYEES, PEOPLE, ACCOUNT2, NYC_TAXI, BOOKS, APACHE, WILDCARD, NESTED, WEBLOGS, JSON_TEST], ignore_unavailable=True) docsuite = partial(doctest.DocFileSuite, diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index 1728be74e6..d4f7213736 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -22,6 +22,7 @@ import static org.opensearch.sql.legacy.TestUtils.getGameOfThronesIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getGeopointIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getJoinTypeIndexMapping; +import static org.opensearch.sql.legacy.TestUtils.getJsonTestIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getLocationIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getMappingFile; import static org.opensearch.sql.legacy.TestUtils.getNestedSimpleIndexMapping; @@ -745,7 +746,12 @@ public enum Index { TestsConstants.TEST_INDEX_GEOPOINT, "dates", getGeopointIndexMapping(), - "src/test/resources/geopoints.json"); + "src/test/resources/geopoints.json"), + JSON_TEST( + TestsConstants.TEST_INDEX_JSON_TEST, + "json", + getJsonTestIndexMapping(), + "src/test/resources/json_test.json"); private final String name; private final String type; diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java index 195dda0cbd..610ad1366a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java @@ -250,6 +250,11 @@ public static String getGeopointIndexMapping() { return getMappingFile(mappingFile); } + public static String getJsonTestIndexMapping() { + String mappingFile = "json_test_index_mapping.json"; + return getMappingFile(mappingFile); + } + public static void loadBulk(Client client, String jsonPath, String defaultIndex) throws Exception { System.out.println(String.format("Loading file %s into opensearch cluster", jsonPath)); diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java index 1e336f544e..387054ac7e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java @@ -58,6 +58,7 @@ public class TestsConstants { public static final String TEST_INDEX_MULTI_NESTED_TYPE = TEST_INDEX + "_multi_nested"; public static final String TEST_INDEX_NESTED_WITH_NULLS = TEST_INDEX + "_nested_with_nulls"; public static final String TEST_INDEX_GEOPOINT = TEST_INDEX + "_geopoint"; + public static final String TEST_INDEX_JSON_TEST = TEST_INDEX + "_json_test"; public static final String DATASOURCES = ".ql-datasources"; public static final String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java new file mode 100644 index 0000000000..62e7868b41 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java @@ -0,0 +1,65 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import org.json.JSONObject; + +import javax.json.Json; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_JSON_TEST; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; + +public class JsonFunctionIT extends PPLIntegTestCase { + @Override + public void init() throws IOException { + loadIndex(Index.JSON_TEST); + } + + @Test + public void test_json_valid() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | where json_valid(json_string) | fields test_name", + TEST_INDEX_JSON_TEST + ) + ); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows( + result, + rows("json object"), + rows("json array"), + rows("json scalar string"), + rows("json empty string") + ); + } + + @Test + public void test_not_json_valid() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | where not json_valid(json_string) | fields test_name", + TEST_INDEX_JSON_TEST + ) + ); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows( + result, + rows("json invalid object") + ); + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json b/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json new file mode 100644 index 0000000000..b825254b11 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json @@ -0,0 +1,12 @@ +{ + "mappings": { + "properties": { + "test_name": { + "type": "text" + }, + "json_string": { + "type": "text" + } + } + } +} diff --git a/integ-test/src/test/resources/json_test.json b/integ-test/src/test/resources/json_test.json new file mode 100644 index 0000000000..e198eb7c43 --- /dev/null +++ b/integ-test/src/test/resources/json_test.json @@ -0,0 +1,10 @@ +{"index":{"_id":"1"}} +{"test_name":"json object", "json_string":"{\"a\":\"1\",\"b\":\"2\"}"} +{"index":{"_id":"2"}} +{"test_name":"json array", "json_string":"[1, 2, 3, 4]"} +{"index":{"_id":"3"}} +{"test_name":"json scalar string", "json_string":"\"abc\""} +{"index":{"_id":"4"}} +{"test_name":"json empty string","json_string":""} +{"index":{"_id":"5"}} +{"test_name":"json invalid object", "json_string":"{\"invalid\":\"json\", \"string\"}"}