From cbcc0f21e123198d121ee6d0ffc6d9e3e9a3bd9e Mon Sep 17 00:00:00 2001 From: joe maley Date: Wed, 15 Jul 2020 18:08:52 -0400 Subject: [PATCH] Fix Dimension::splitting_value for signed and negative inputs (#1725) The bitwise calculation for the partitioner's dimension splitting value uses an extra high to avoid overflow mid-calculation. The types may be signed or unsigned. If the input arguments are signed and negative, we need to flip the high bit to maintain two's complement notation. Co-authored-by: Joe Maley --- tiledb/sm/array_schema/dimension.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tiledb/sm/array_schema/dimension.cc b/tiledb/sm/array_schema/dimension.cc index 53fd0f972fb..75dc4ee5727 100644 --- a/tiledb/sm/array_schema/dimension.cc +++ b/tiledb/sm/array_schema/dimension.cc @@ -889,6 +889,12 @@ void Dimension::splitting_value( std::bitset<65> r_t0(r_t[0]); std::bitset<65> r_t1(r_t[1]); + // If `T` is signed, respect two's complement on the high bit. + if (std::is_signed::value) { + r_t0[64] = r_t0[63]; + r_t1[64] = r_t1[63]; + } + // Subtract `r_t0` from `r_t1`. while (r_t0 != 0) { const std::bitset<65> carry = (~r_t1) & r_t0;