Skip to content

Commit

Permalink
Allow passing int for uint spec when int precision > uint precision (#…
Browse files Browse the repository at this point in the history 
…298)
  • Loading branch information
@rly
rly authored Feb 20, 2020
1 parent d432709 commit b395192
Show file tree
Hide file tree
Showing 2 changed files with 195 additions and 42 deletions.
11 changes: 10 additions & 1 deletion src/hdmf/build/objectmapper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,9 +127,18 @@ def __resolve_dtype(cls, given, specified):
g = np.dtype(given)
s = np.dtype(specified)
if g.itemsize <= s.itemsize:
if g.name[:3] != s.name[:3]: # different types
warnings.warn('Value with data type %s is being converted to data type %s as specified.'
% (g.name, s.name))
return s.type
else:
if g.name[:3] != s.name[:3]: # different types
if s.name.startswith('uint') and (g.name.startswith('int') or g.name.startswith('float')):
# e.g., given int64 and spec uint32, return uint64. given float32 and spec uint8, return uint32.
warnings.warn('Value with data type %s is being converted to data type %s as specified.'
% (g.name, s.name))
return np.dtype('uint' + str(int(g.itemsize*8))).type
if g.name[:3] != s.name[:3]: # different types
# if specified int and given uint/float or if specified float and given int/uint
if s.itemsize < 8:
msg = "expected %s, received %s - must supply %s or higher precision" % (s.name, g.name, s.name)
else:
Expand Down
226 changes: 185 additions & 41 deletions tests/unit/build_tests/test_io_map.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -575,71 +575,215 @@ def test_value_none(self):
spec = DatasetSpec('an example dataset', RefSpec(reftype='object', target_type='int'), name='data')
self.assertTupleEqual(ObjectMapper.convert_dtype(spec, None), (None, 'object'))

def test_convert_higher_precision(self):
"""Test that passing a data type with a precision <= specified returns the higher precision type"""
# do full matrix test of given value x and spec y, what does convert_dtype return?
def test_convert_to_64bit_spec(self):
"""
Test that if given any value for a spec with a 64-bit dtype, convert_dtype will convert to the spec type.
Also test that if the given base type differs from the spec base type, a warning is raised.
"""
spec_type = 'float64'
value_types = ['float', 'float32', 'double', 'float64']
self.convert_higher_precision_helper(spec_type, value_types)
value_types = ['double', 'float64', 'float', 'float32']
self._test_convert_higher_precision_helper(spec_type, value_types)

spec_type = 'float64'
value_types = ['long', 'int64', 'int', 'int32', 'int16', 'int8', 'uint64', 'uint', 'uint32', 'uint16', 'uint8']
self._test_convert_higher_precision_helper_warn(spec_type, value_types)

spec_type = 'int64'
value_types = ['long', 'int64', 'uint64', 'int', 'int32', 'int16', 'int8']
self.convert_higher_precision_helper(spec_type, value_types)
value_types = ['long', 'int64', 'int', 'int32', 'int16', 'int8']
self._test_convert_higher_precision_helper(spec_type, value_types)

spec_type = 'int32'
value_types = ['int32', 'int16', 'int8']
self.convert_higher_precision_helper(spec_type, value_types)
spec_type = 'int64'
value_types = ['double', 'float64', 'float', 'float32', 'uint64', 'uint', 'uint32', 'uint16', 'uint8']
self._test_convert_higher_precision_helper_warn(spec_type, value_types)

spec_type = 'uint64'
value_types = ['uint64', 'uint', 'uint32', 'uint16', 'uint8']
self._test_convert_higher_precision_helper(spec_type, value_types)

spec_type = 'uint64'
value_types = ['double', 'float64', 'float', 'float32', 'long', 'int64', 'int', 'int32', 'int16', 'int8']
self._test_convert_higher_precision_helper_warn(spec_type, value_types)

def test_convert_to_float32_spec(self):
"""Test conversion of various types to float32.
If given a value with precision > float32 and float base type, convert_dtype will keep the higher precision.
If given a value with precision > float32 and different base type, convert_dtype will raise a ValueError.
If given a value with precision <= float32 and float base type, convert_dtype will convert to float32.
If given a value with precision <= float32 and different base type, convert_dtype will convert to float32 and
raise a warning.
"""
spec_type = 'float32'
value_types = ['double', 'float64']
self._test_keep_higher_precision_helper(spec_type, value_types)

value_types = ['long', 'int64', 'uint64']
self._test_convert_mismatch_helper(spec_type, value_types)

value_types = ['float', 'float32']
self._test_convert_higher_precision_helper(spec_type, value_types)

value_types = ['int', 'int32', 'int16', 'int8', 'uint', 'uint32', 'uint16', 'uint8']
self._test_convert_higher_precision_helper_warn(spec_type, value_types)

def test_convert_to_int32_spec(self):
"""Test conversion of various types to int32.
If given a value with precision > int32 and int base type, convert_dtype will keep the higher precision.
If given a value with precision > int32 and different base type, convert_dtype will raise a ValueError.
If given a value with precision <= int32 and int base type, convert_dtype will convert to int32.
If given a value with precision <= int32 and different base type, convert_dtype will convert to int32 and
raise a warning.
"""
spec_type = 'int32'
value_types = ['int64', 'long']
self._test_keep_higher_precision_helper(spec_type, value_types)

value_types = ['double', 'float64', 'uint64']
self._test_convert_mismatch_helper(spec_type, value_types)

value_types = ['int', 'int32', 'int16', 'int8', ]
self._test_convert_higher_precision_helper(spec_type, value_types)

value_types = ['float', 'float32', 'uint', 'uint32', 'uint16', 'uint8']
self._test_convert_higher_precision_helper_warn(spec_type, value_types)

def test_convert_to_uint32_spec(self):
"""Test conversion of various types to uint32.
If given a value with precision > uint32 and uint base type, convert_dtype will keep the higher precision.
If given a value with precision > uint32 and different base type, convert_dtype will convert to uint with the
original precision.
If given a value with precision <= uint32 and uint base type, convert_dtype will convert to uint32.
If given a value with precision <= uint32 and different base type, convert_dtype will convert to uint32 and
raise a warning.
"""
spec_type = 'uint32'
value_types = ['uint64']
self._test_keep_higher_precision_helper(spec_type, value_types)

value_types = ['double', 'float64', 'long', 'int64']
expected_types = ['uint64', 'uint64', 'uint64', 'uint64']
self._test_convert_uint_helper(spec_type, value_types, expected_types)

value_types = ['uint', 'uint32', 'uint16', 'uint8']
self._test_convert_higher_precision_helper(spec_type, value_types)

value_types = ['float', 'float32', 'int', 'int32', 'int16', 'int8']
self._test_convert_higher_precision_helper_warn(spec_type, value_types)

def test_convert_to_int16_spec(self):
"""Test conversion of various types to int16.
If given a value with precision > int16 and int base type, convert_dtype will keep the higher precision.
If given a value with precision > int16 and different base type, convert_dtype will raise a ValueError.
If given a value with precision <= int16 and int base type, convert_dtype will convert to int16.
If given a value with precision <= int16 and different base type, convert_dtype will convert to int16 and
raise a warning.
"""
spec_type = 'int16'
value_types = ['long', 'int64', 'int', 'int32']
self._test_keep_higher_precision_helper(spec_type, value_types)

value_types = ['double', 'float64', 'float', 'float32', 'uint64', 'uint', 'uint32']
self._test_convert_mismatch_helper(spec_type, value_types)

value_types = ['int16', 'int8']
self.convert_higher_precision_helper(spec_type, value_types)
self._test_convert_higher_precision_helper(spec_type, value_types)

value_types = ['uint16', 'uint8']
self._test_convert_higher_precision_helper_warn(spec_type, value_types)

def test_convert_to_uint16_spec(self):
"""Test conversion of various types to uint16.
If given a value with precision > uint16 and uint base type, convert_dtype will keep the higher precision.
If given a value with precision > uint16 and different base type, convert_dtype will convert to uint with the
original precision.
If given a value with precision <= uint16 and uint base type, convert_dtype will convert to uint16.
If given a value with precision <= uint16 and different base type, convert_dtype will convert to uint16 and
raise a warning.
"""
spec_type = 'uint16'
value_types = ['uint64', 'uint', 'uint32']
self._test_keep_higher_precision_helper(spec_type, value_types)

value_types = ['double', 'float64', 'long', 'int64']
expected_types = ['uint64', 'uint64', 'uint64', 'uint64']
self._test_convert_uint_helper(spec_type, value_types, expected_types)

value_types = ['float', 'float32', 'int', 'int32']
expected_types = ['uint32', 'uint32', 'uint32', 'uint32']
self._test_convert_uint_helper(spec_type, value_types, expected_types)

value_types = ['uint16', 'uint8']
self._test_convert_higher_precision_helper(spec_type, value_types)

spec_type = 'uint32'
value_types = ['uint32', 'uint16', 'uint8']
self.convert_higher_precision_helper(spec_type, value_types)
value_types = ['int16', 'int8']
self._test_convert_higher_precision_helper_warn(spec_type, value_types)

def _get_type(self, type_str):
return ObjectMapper._ObjectMapper__dtypes[type_str] # apply ObjectMapper mapping string to dtype

def convert_higher_precision_helper(self, spec_type, value_types):
def _test_convert_higher_precision_helper(self, spec_type, value_types):
data = 2
spec = DatasetSpec('an example dataset', spec_type, name='data')
match = (np.dtype(spec_type).type(data), np.dtype(spec_type))
match = (self._get_type(spec_type)(data), self._get_type(spec_type))
for dtype in value_types:
value = np.dtype(dtype).type(data)
value = self._get_type(dtype)(data) # convert data to given dtype
with self.subTest(dtype=dtype):
ret = ObjectMapper.convert_dtype(spec, value)
self.assertTupleEqual(ret, match)
self.assertIs(ret[0].dtype, match[1])

def test_keep_higher_precision(self):
"""Test that passing a data type with a precision >= specified return the given type"""
spec_type = 'float'
value_types = ['double', 'float64']
self.keep_higher_precision_helper(spec_type, value_types)
self.assertIs(ret[0].dtype.type, match[1])

spec_type = 'int'
value_types = ['int64']
self.keep_higher_precision_helper(spec_type, value_types)

spec_type = 'int8'
value_types = ['long', 'int64', 'int', 'int32', 'int16']
self.keep_higher_precision_helper(spec_type, value_types)

spec_type = 'uint'
value_types = ['uint64']
self.keep_higher_precision_helper(spec_type, value_types)
def _test_convert_higher_precision_helper_warn(self, spec_type, value_types):
data = 2
spec = DatasetSpec('an example dataset', spec_type, name='data')
match = (self._get_type(spec_type)(data), self._get_type(spec_type))
for dtype in value_types:
value = self._get_type(dtype)(data) # convert data to given dtype
with self.subTest(dtype=dtype):
s = np.dtype(self._get_type(spec_type))
g = np.dtype(self._get_type(dtype))
msg = 'Value with data type %s is being converted to data type %s as specified.' % (g.name, s.name)
with self.assertWarnsWith(UserWarning, msg):
ret = ObjectMapper.convert_dtype(spec, value)
self.assertTupleEqual(ret, match)
self.assertIs(ret[0].dtype.type, match[1])

spec_type = 'uint8'
value_types = ['uint64', 'uint32', 'uint', 'uint16']
self.keep_higher_precision_helper(spec_type, value_types)
def _test_convert_mismatch_helper(self, spec_type, value_types):
data = 2
spec = DatasetSpec('an example dataset', spec_type, name='data')
for dtype in value_types:
value = self._get_type(dtype)(data) # convert data to given dtype
with self.subTest(dtype=dtype):
s = np.dtype(self._get_type(spec_type))
g = np.dtype(self._get_type(dtype))
msg = r"expected %s, received %s - must supply %s.*" % (s.name, g.name, s.name)
with self.assertRaisesRegex(ValueError, msg):
ObjectMapper.convert_dtype(spec, value)

def keep_higher_precision_helper(self, spec_type, value_types):
def _test_keep_higher_precision_helper(self, spec_type, value_types):
data = 2
spec = DatasetSpec('an example dataset', spec_type, name='data')
for dtype in value_types:
value = np.dtype(dtype).type(data)
match = (value, np.dtype(dtype))
value = self._get_type(dtype)(data)
match = (value, self._get_type(dtype))
with self.subTest(dtype=dtype):
ret = ObjectMapper.convert_dtype(spec, value)
self.assertTupleEqual(ret, match)
self.assertIs(ret[0].dtype, match[1])
self.assertIs(ret[0].dtype.type, match[1])

def _test_convert_uint_helper(self, spec_type, value_types, expected_types):
data = 2
spec = DatasetSpec('an example dataset', spec_type, name='data')
for dtype, exp_type in zip(value_types, expected_types):
value = self._get_type(dtype)(data) # convert data to given dtype
match = (self._get_type(exp_type)(data), self._get_type(exp_type))
with self.subTest(dtype=dtype):
s = np.dtype(self._get_type(spec_type))
g = np.dtype(self._get_type(dtype))
msg = 'Value with data type %s is being converted to data type %s as specified.' % (g.name, s.name)
with self.assertWarnsWith(UserWarning, msg):
ret = ObjectMapper.convert_dtype(spec, value)
self.assertTupleEqual(ret, match)
self.assertIs(ret[0].dtype.type, match[1])

def test_no_spec(self):
spec_type = None
Expand Down

0 comments on commit b395192

Please sign in to comment.