Fix JsonExtractScalar when no value is extracted (#9500)

apache · Sep 30, 2022 · db92385 · db92385
1 parent 270315a
commit db92385
Show file tree

Hide file tree

Showing 5 changed files with 212 additions and 56 deletions.
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/utils/PinotDataType.java b/pinot-common/src/main/java/org/apache/pinot/common/utils/PinotDataType.java
@@ -966,7 +966,7 @@ public String toJson(Object value) {
     if (value instanceof String) {
       try {
         // Try to parse the string as JSON first
-        return JsonUtils.stringToJsonNode((String) value).toString();
+        return JsonUtils.stringToJsonNodeWithBigDecimal((String) value).toString();
       } catch (JsonParseException jpe) {
         // String does not represent a well-formed JSON. Ignore this exception because we are going to try to convert
         // Java String object to JSON string.

diff --git a/...-core/src/main/java/org/apache/pinot/core/common/evaluators/DefaultJsonPathEvaluator.java b/...-core/src/main/java/org/apache/pinot/core/common/evaluators/DefaultJsonPathEvaluator.java
@@ -42,8 +42,8 @@
 public final class DefaultJsonPathEvaluator implements JsonPathEvaluator {
 
   // This ObjectMapper requires special configurations, hence we can't use pinot JsonUtils here.
-  private static final ObjectMapper OBJECT_MAPPER_WITH_BIG_DECIMAL = new ObjectMapper()
-      .configure(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS, true);
+  private static final ObjectMapper OBJECT_MAPPER_WITH_BIG_DECIMAL =
+      new ObjectMapper().enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS);
 
   private static final ParseContext JSON_PARSER_CONTEXT = JsonPath.using(
       new Configuration.ConfigurationBuilder().jsonProvider(new JacksonJsonProvider())
@@ -423,40 +423,80 @@ public <T extends ForwardIndexReaderContext> void evaluateBlock(int[] docIds, in
     }
   }
 
+  @Nullable
   private <T> T extractFromBytes(Dictionary dictionary, int dictId) {
-    return JSON_PARSER_CONTEXT.parseUtf8(dictionary.getBytesValue(dictId)).read(_jsonPath);
+    try {
+      return JSON_PARSER_CONTEXT.parseUtf8(dictionary.getBytesValue(dictId)).read(_jsonPath);
+    } catch (Exception e) {
+      return null;
+    }
   }
 
+  @Nullable
   private <T, R extends ForwardIndexReaderContext> T extractFromBytes(ForwardIndexReader<R> reader, R context,
       int docId) {
-    return JSON_PARSER_CONTEXT.parseUtf8(reader.getBytes(docId, context)).read(_jsonPath);
+    try {
+      return JSON_PARSER_CONTEXT.parseUtf8(reader.getBytes(docId, context)).read(_jsonPath);
+    } catch (Exception e) {
+      return null;
+    }
   }
 
+  @Nullable
   private <T> T extractFromBytesWithExactBigDecimal(Dictionary dictionary, int dictId) {
-    return JSON_PARSER_CONTEXT_WITH_BIG_DECIMAL.parseUtf8(dictionary.getBytesValue(dictId)).read(_jsonPath);
+    try {
+      return JSON_PARSER_CONTEXT_WITH_BIG_DECIMAL.parseUtf8(dictionary.getBytesValue(dictId)).read(_jsonPath);
+    } catch (Exception e) {
+      return null;
+    }
   }
 
+  @Nullable
   private <R extends ForwardIndexReaderContext> BigDecimal extractFromBytesWithExactBigDecimal(
       ForwardIndexReader<R> reader, R context, int docId) {
-    return JSON_PARSER_CONTEXT_WITH_BIG_DECIMAL.parseUtf8(reader.getBytes(docId, context)).read(_jsonPath);
+    try {
+      return JSON_PARSER_CONTEXT_WITH_BIG_DECIMAL.parseUtf8(reader.getBytes(docId, context)).read(_jsonPath);
+    } catch (Exception e) {
+      return null;
+    }
   }
 
+  @Nullable
   private <T> T extractFromString(Dictionary dictionary, int dictId) {
-    return JSON_PARSER_CONTEXT.parse(dictionary.getStringValue(dictId)).read(_jsonPath);
+    try {
+      return JSON_PARSER_CONTEXT.parse(dictionary.getStringValue(dictId)).read(_jsonPath);
+    } catch (Exception e) {
+      return null;
+    }
   }
 
+  @Nullable
   private <T, R extends ForwardIndexReaderContext> T extractFromString(ForwardIndexReader<R> reader, R context,
       int docId) {
-    return JSON_PARSER_CONTEXT.parseUtf8(reader.getBytes(docId, context)).read(_jsonPath);
+    try {
+      return JSON_PARSER_CONTEXT.parseUtf8(reader.getBytes(docId, context)).read(_jsonPath);
+    } catch (Exception e) {
+      return null;
+    }
   }
 
+  @Nullable
   private <T> T extractFromStringWithExactBigDecimal(Dictionary dictionary, int dictId) {
-    return JSON_PARSER_CONTEXT_WITH_BIG_DECIMAL.parse(dictionary.getStringValue(dictId)).read(_jsonPath);
+    try {
+      return JSON_PARSER_CONTEXT_WITH_BIG_DECIMAL.parse(dictionary.getStringValue(dictId)).read(_jsonPath);
+    } catch (Exception e) {
+      return null;
+    }
   }
 
+  @Nullable
   private <R extends ForwardIndexReaderContext> BigDecimal extractFromStringWithExactBigDecimal(
       ForwardIndexReader<R> reader, R context, int docId) {
-    return JSON_PARSER_CONTEXT_WITH_BIG_DECIMAL.parseUtf8(reader.getBytes(docId, context)).read(_jsonPath);
+    try {
+      return JSON_PARSER_CONTEXT_WITH_BIG_DECIMAL.parseUtf8(reader.getBytes(docId, context)).read(_jsonPath);
+    } catch (Exception e) {
+      return null;
+    }
   }
 
   private void processValue(int index, Object value, int defaultValue, int[] valueBuffer) {

diff --git a/...est/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunctionTest.java b/...est/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunctionTest.java
@@ -88,6 +88,7 @@ public abstract class BaseTransformFunctionTest {
   protected static final String TIME_COLUMN = "timeColumn";
   protected static final String TIMESTAMP_COLUMN = "timestampColumn";
   protected static final String JSON_COLUMN = "json";
+  protected static final String DEFAULT_JSON_COLUMN = "defaultJson";
   protected final int[] _intSVValues = new int[NUM_ROWS];
   protected final long[] _longSVValues = new long[NUM_ROWS];
   protected final float[] _floatSVValues = new float[NUM_ROWS];
@@ -184,7 +185,8 @@ public void setUp()
         .addSingleValueDimension(STRING_SV_COLUMN, FieldSpec.DataType.STRING)
         .addSingleValueDimension(STRING_ALPHANUM_SV_COLUMN, FieldSpec.DataType.STRING)
         .addSingleValueDimension(BYTES_SV_COLUMN, FieldSpec.DataType.BYTES)
-        .addSingleValueDimension(JSON_COLUMN, FieldSpec.DataType.STRING, Integer.MAX_VALUE, null)
+        .addSingleValueDimension(JSON_COLUMN, FieldSpec.DataType.JSON)
+        .addSingleValueDimension(DEFAULT_JSON_COLUMN, FieldSpec.DataType.JSON)
         .addMultiValueDimension(INT_MV_COLUMN, FieldSpec.DataType.INT)
         .addMultiValueDimension(LONG_MV_COLUMN, FieldSpec.DataType.LONG)
         .addMultiValueDimension(FLOAT_MV_COLUMN, FieldSpec.DataType.FLOAT)