-
Notifications
You must be signed in to change notification settings - Fork 176
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
fix: skip negative scale checks for creating decimals #723
Changes from all commits
d8f2946
d00e05c
d0b8ec8
027f9b8
2230c66
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -45,6 +45,19 @@ public abstract class CometVector extends ColumnVector { | |
private final byte[] DECIMAL_BYTES = new byte[DECIMAL_BYTE_WIDTH]; | ||
protected final boolean useDecimal128; | ||
|
||
private static final long decimalValOffset; | ||
|
||
static { | ||
try { | ||
java.lang.reflect.Field unsafeField = sun.misc.Unsafe.class.getDeclaredField("theUnsafe"); | ||
unsafeField.setAccessible(true); | ||
final sun.misc.Unsafe unsafe = (sun.misc.Unsafe) unsafeField.get(null); | ||
decimalValOffset = unsafe.objectFieldOffset(Decimal.class.getDeclaredField("decimalVal")); | ||
} catch (Throwable e) { | ||
throw new RuntimeException(e); | ||
} | ||
} | ||
|
||
protected CometVector(DataType type, boolean useDecimal128) { | ||
super(type); | ||
this.useDecimal128 = useDecimal128; | ||
|
@@ -73,31 +86,35 @@ public boolean isFixedLength() { | |
@Override | ||
public Decimal getDecimal(int i, int precision, int scale) { | ||
if (!useDecimal128 && precision <= Decimal.MAX_INT_DIGITS() && type instanceof IntegerType) { | ||
return Decimal.createUnsafe(getInt(i), precision, scale); | ||
return createDecimal(getInt(i), precision, scale); | ||
} else if (!useDecimal128 && precision <= Decimal.MAX_LONG_DIGITS()) { | ||
return Decimal.createUnsafe(getLong(i), precision, scale); | ||
return createDecimal(getLong(i), precision, scale); | ||
} else { | ||
byte[] bytes = getBinaryDecimal(i); | ||
BigInteger bigInteger = new BigInteger(bytes); | ||
BigDecimal javaDecimal = new BigDecimal(bigInteger, scale); | ||
try { | ||
return Decimal.apply(javaDecimal, precision, scale); | ||
} catch (ArithmeticException e) { | ||
throw new ArithmeticException( | ||
"Cannot convert " | ||
+ javaDecimal | ||
+ " (bytes: " | ||
+ bytes | ||
+ ", integer: " | ||
+ bigInteger | ||
+ ") to decimal with precision: " | ||
+ precision | ||
+ " and scale: " | ||
+ scale); | ||
} | ||
return createDecimal(javaDecimal, precision, scale); | ||
} | ||
} | ||
|
||
/** This method skips the negative scale check, otherwise the same as Decimal.createUnsafe(). */ | ||
private Decimal createDecimal(long unscaled, int precision, int scale) { | ||
Decimal dec = new Decimal(); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Seems the check for negative scale is a feature for ANSI correctness? https://issues.apache.org/jira/browse/SPARK-30252 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It does not need to do the check every read, we are running ANSI tests in spark_sql_test_ansi.yml |
||
dec.org$apache$spark$sql$types$Decimal$$longVal_$eq(unscaled); | ||
dec.org$apache$spark$sql$types$Decimal$$_precision_$eq(precision); | ||
dec.org$apache$spark$sql$types$Decimal$$_scale_$eq(scale); | ||
return dec; | ||
} | ||
|
||
/** This method skips a few checks, otherwise the same as Decimal.apply(). */ | ||
private Decimal createDecimal(BigDecimal value, int precision, int scale) { | ||
Decimal dec = new Decimal(); | ||
Platform.putObjectVolatile(dec, decimalValOffset, new scala.math.BigDecimal(value)); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The spark version has some additional checks for rounding and precision. Do we not need those?
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. No we do not need the precision check because it is done in advance There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Just for my knowledge, where? |
||
dec.org$apache$spark$sql$types$Decimal$$_precision_$eq(precision); | ||
dec.org$apache$spark$sql$types$Decimal$$_scale_$eq(scale); | ||
return dec; | ||
} | ||
|
||
/** | ||
* Reads a 16-byte byte array which are encoded big-endian for decimal128 into internal byte | ||
* array. | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is not directly related but forgot to add based on #705 (comment)