Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow passing int for uint spec when int precision > uint precision #298

Merged
merged 3 commits into from
Feb 20, 2020
Merged

Allow passing int for uint spec when int precision > uint precision #298

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4724d96
Allow passing int for uint spec when int precision > uint precision
rly Feb 19, 2020
721bf72
Improve comments, fix flake8
rly Feb 19, 2020
2676646
Fix - do not halve the precision for int/float to uint, add warning
rly Feb 19, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion src/hdmf/build/objectmapper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,9 +127,18 @@ def __resolve_dtype(cls, given, specified):
g = np.dtype(given)
s = np.dtype(specified)
if g.itemsize <= s.itemsize:
if g.name[:3] != s.name[:3]: # different types
warnings.warn('Value with data type %s is being converted to data type %s as specified.'
% (g.name, s.name))
return s.type
else:
if g.name[:3] != s.name[:3]: # different types
if s.name.startswith('uint') and (g.name.startswith('int') or g.name.startswith('float')):
# e.g., given int64 and spec uint32, return uint64. given float32 and spec uint8, return uint32.
warnings.warn('Value with data type %s is being converted to data type %s as specified.'
% (g.name, s.name))
return np.dtype('uint' + str(int(g.itemsize*8))).type
if g.name[:3] != s.name[:3]: # different types
# if specified int and given uint/float or if specified float and given int/uint
if s.itemsize < 8:
msg = "expected %s, received %s - must supply %s or higher precision" % (s.name, g.name, s.name)
else:
Expand Down
226 changes: 185 additions & 41 deletions tests/unit/build_tests/test_io_map.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -575,71 +575,215 @@ def test_value_none(self):
spec = DatasetSpec('an example dataset', RefSpec(reftype='object', target_type='int'), name='data')
self.assertTupleEqual(ObjectMapper.convert_dtype(spec, None), (None, 'object'))

def test_convert_higher_precision(self):
"""Test that passing a data type with a precision <= specified returns the higher precision type"""
# do full matrix test of given value x and spec y, what does convert_dtype return?
def test_convert_to_64bit_spec(self):
"""
Test that if given any value for a spec with a 64-bit dtype, convert_dtype will convert to the spec type.
Also test that if the given base type differs from the spec base type, a warning is raised.
"""
spec_type = 'float64'
value_types = ['float', 'float32', 'double', 'float64']
self.convert_higher_precision_helper(spec_type, value_types)
value_types = ['double', 'float64', 'float', 'float32']
self._test_convert_higher_precision_helper(spec_type, value_types)

spec_type = 'float64'
value_types = ['long', 'int64', 'int', 'int32', 'int16', 'int8', 'uint64', 'uint', 'uint32', 'uint16', 'uint8']
self._test_convert_higher_precision_helper_warn(spec_type, value_types)

spec_type = 'int64'
value_types = ['long', 'int64', 'uint64', 'int', 'int32', 'int16', 'int8']
self.convert_higher_precision_helper(spec_type, value_types)
value_types = ['long', 'int64', 'int', 'int32', 'int16', 'int8']
self._test_convert_higher_precision_helper(spec_type, value_types)

spec_type = 'int32'
value_types = ['int32', 'int16', 'int8']
self.convert_higher_precision_helper(spec_type, value_types)
spec_type = 'int64'
value_types = ['double', 'float64', 'float', 'float32', 'uint64', 'uint', 'uint32', 'uint16', 'uint8']
self._test_convert_higher_precision_helper_warn(spec_type, value_types)

spec_type = 'uint64'
value_types = ['uint64', 'uint', 'uint32', 'uint16', 'uint8']
self._test_convert_higher_precision_helper(spec_type, value_types)

spec_type = 'uint64'
value_types = ['double', 'float64', 'float', 'float32', 'long', 'int64', 'int', 'int32', 'int16', 'int8']
self._test_convert_higher_precision_helper_warn(spec_type, value_types)

def test_convert_to_float32_spec(self):
"""Test conversion of various types to float32.
If given a value with precision > float32 and float base type, convert_dtype will keep the higher precision.
If given a value with precision > float32 and different base type, convert_dtype will raise a ValueError.
If given a value with precision <= float32 and float base type, convert_dtype will convert to float32.
If given a value with precision <= float32 and different base type, convert_dtype will convert to float32 and
raise a warning.
"""
spec_type = 'float32'
value_types = ['double', 'float64']
self._test_keep_higher_precision_helper(spec_type, value_types)

value_types = ['long', 'int64', 'uint64']
self._test_convert_mismatch_helper(spec_type, value_types)

value_types = ['float', 'float32']
self._test_convert_higher_precision_helper(spec_type, value_types)

value_types = ['int', 'int32', 'int16', 'int8', 'uint', 'uint32', 'uint16', 'uint8']
self._test_convert_higher_precision_helper_warn(spec_type, value_types)

def test_convert_to_int32_spec(self):
"""Test conversion of various types to int32.
If given a value with precision > int32 and int base type, convert_dtype will keep the higher precision.
If given a value with precision > int32 and different base type, convert_dtype will raise a ValueError.
If given a value with precision <= int32 and int base type, convert_dtype will convert to int32.
If given a value with precision <= int32 and different base type, convert_dtype will convert to int32 and
raise a warning.
"""
spec_type = 'int32'
value_types = ['int64', 'long']
self._test_keep_higher_precision_helper(spec_type, value_types)

value_types = ['double', 'float64', 'uint64']
self._test_convert_mismatch_helper(spec_type, value_types)

value_types = ['int', 'int32', 'int16', 'int8', ]
self._test_convert_higher_precision_helper(spec_type, value_types)

value_types = ['float', 'float32', 'uint', 'uint32', 'uint16', 'uint8']
self._test_convert_higher_precision_helper_warn(spec_type, value_types)

def test_convert_to_uint32_spec(self):
"""Test conversion of various types to uint32.
If given a value with precision > uint32 and uint base type, convert_dtype will keep the higher precision.
If given a value with precision > uint32 and different base type, convert_dtype will convert to uint with the
original precision.
If given a value with precision <= uint32 and uint base type, convert_dtype will convert to uint32.
If given a value with precision <= uint32 and different base type, convert_dtype will convert to uint32 and
raise a warning.
"""
spec_type = 'uint32'
value_types = ['uint64']
self._test_keep_higher_precision_helper(spec_type, value_types)

value_types = ['double', 'float64', 'long', 'int64']
expected_types = ['uint64', 'uint64', 'uint64', 'uint64']
self._test_convert_uint_helper(spec_type, value_types, expected_types)

value_types = ['uint', 'uint32', 'uint16', 'uint8']
self._test_convert_higher_precision_helper(spec_type, value_types)

value_types = ['float', 'float32', 'int', 'int32', 'int16', 'int8']
self._test_convert_higher_precision_helper_warn(spec_type, value_types)

def test_convert_to_int16_spec(self):
"""Test conversion of various types to int16.
If given a value with precision > int16 and int base type, convert_dtype will keep the higher precision.
If given a value with precision > int16 and different base type, convert_dtype will raise a ValueError.
If given a value with precision <= int16 and int base type, convert_dtype will convert to int16.
If given a value with precision <= int16 and different base type, convert_dtype will convert to int16 and
raise a warning.
"""
spec_type = 'int16'
value_types = ['long', 'int64', 'int', 'int32']
self._test_keep_higher_precision_helper(spec_type, value_types)

value_types = ['double', 'float64', 'float', 'float32', 'uint64', 'uint', 'uint32']
self._test_convert_mismatch_helper(spec_type, value_types)

value_types = ['int16', 'int8']
self.convert_higher_precision_helper(spec_type, value_types)
self._test_convert_higher_precision_helper(spec_type, value_types)

value_types = ['uint16', 'uint8']
self._test_convert_higher_precision_helper_warn(spec_type, value_types)

def test_convert_to_uint16_spec(self):
"""Test conversion of various types to uint16.
If given a value with precision > uint16 and uint base type, convert_dtype will keep the higher precision.
If given a value with precision > uint16 and different base type, convert_dtype will convert to uint with the
original precision.
If given a value with precision <= uint16 and uint base type, convert_dtype will convert to uint16.
If given a value with precision <= uint16 and different base type, convert_dtype will convert to uint16 and
raise a warning.
"""
spec_type = 'uint16'
value_types = ['uint64', 'uint', 'uint32']
self._test_keep_higher_precision_helper(spec_type, value_types)

value_types = ['double', 'float64', 'long', 'int64']
expected_types = ['uint64', 'uint64', 'uint64', 'uint64']
self._test_convert_uint_helper(spec_type, value_types, expected_types)

value_types = ['float', 'float32', 'int', 'int32']
expected_types = ['uint32', 'uint32', 'uint32', 'uint32']
self._test_convert_uint_helper(spec_type, value_types, expected_types)

value_types = ['uint16', 'uint8']
self._test_convert_higher_precision_helper(spec_type, value_types)

spec_type = 'uint32'
value_types = ['uint32', 'uint16', 'uint8']
self.convert_higher_precision_helper(spec_type, value_types)
value_types = ['int16', 'int8']
self._test_convert_higher_precision_helper_warn(spec_type, value_types)

def _get_type(self, type_str):
return ObjectMapper._ObjectMapper__dtypes[type_str] # apply ObjectMapper mapping string to dtype

def convert_higher_precision_helper(self, spec_type, value_types):
def _test_convert_higher_precision_helper(self, spec_type, value_types):
data = 2
spec = DatasetSpec('an example dataset', spec_type, name='data')
match = (np.dtype(spec_type).type(data), np.dtype(spec_type))
match = (self._get_type(spec_type)(data), self._get_type(spec_type))
for dtype in value_types:
value = np.dtype(dtype).type(data)
value = self._get_type(dtype)(data) # convert data to given dtype
with self.subTest(dtype=dtype):
ret = ObjectMapper.convert_dtype(spec, value)
self.assertTupleEqual(ret, match)
self.assertIs(ret[0].dtype, match[1])

def test_keep_higher_precision(self):
"""Test that passing a data type with a precision >= specified return the given type"""
spec_type = 'float'
value_types = ['double', 'float64']
self.keep_higher_precision_helper(spec_type, value_types)
self.assertIs(ret[0].dtype.type, match[1])

spec_type = 'int'
value_types = ['int64']
self.keep_higher_precision_helper(spec_type, value_types)

spec_type = 'int8'
value_types = ['long', 'int64', 'int', 'int32', 'int16']
self.keep_higher_precision_helper(spec_type, value_types)

spec_type = 'uint'
value_types = ['uint64']
self.keep_higher_precision_helper(spec_type, value_types)
def _test_convert_higher_precision_helper_warn(self, spec_type, value_types):
data = 2
spec = DatasetSpec('an example dataset', spec_type, name='data')
match = (self._get_type(spec_type)(data), self._get_type(spec_type))
for dtype in value_types:
value = self._get_type(dtype)(data) # convert data to given dtype
with self.subTest(dtype=dtype):
s = np.dtype(self._get_type(spec_type))
g = np.dtype(self._get_type(dtype))
msg = 'Value with data type %s is being converted to data type %s as specified.' % (g.name, s.name)
with self.assertWarnsWith(UserWarning, msg):
ret = ObjectMapper.convert_dtype(spec, value)
self.assertTupleEqual(ret, match)
self.assertIs(ret[0].dtype.type, match[1])

spec_type = 'uint8'
value_types = ['uint64', 'uint32', 'uint', 'uint16']
self.keep_higher_precision_helper(spec_type, value_types)
def _test_convert_mismatch_helper(self, spec_type, value_types):
data = 2
spec = DatasetSpec('an example dataset', spec_type, name='data')
for dtype in value_types:
value = self._get_type(dtype)(data) # convert data to given dtype
with self.subTest(dtype=dtype):
s = np.dtype(self._get_type(spec_type))
g = np.dtype(self._get_type(dtype))
msg = r"expected %s, received %s - must supply %s.*" % (s.name, g.name, s.name)
with self.assertRaisesRegex(ValueError, msg):
ObjectMapper.convert_dtype(spec, value)

def keep_higher_precision_helper(self, spec_type, value_types):
def _test_keep_higher_precision_helper(self, spec_type, value_types):
data = 2
spec = DatasetSpec('an example dataset', spec_type, name='data')
for dtype in value_types:
value = np.dtype(dtype).type(data)
match = (value, np.dtype(dtype))
value = self._get_type(dtype)(data)
match = (value, self._get_type(dtype))
with self.subTest(dtype=dtype):
ret = ObjectMapper.convert_dtype(spec, value)
self.assertTupleEqual(ret, match)
self.assertIs(ret[0].dtype, match[1])
self.assertIs(ret[0].dtype.type, match[1])

def _test_convert_uint_helper(self, spec_type, value_types, expected_types):
data = 2
spec = DatasetSpec('an example dataset', spec_type, name='data')
for dtype, exp_type in zip(value_types, expected_types):
value = self._get_type(dtype)(data) # convert data to given dtype
match = (self._get_type(exp_type)(data), self._get_type(exp_type))
with self.subTest(dtype=dtype):
s = np.dtype(self._get_type(spec_type))
g = np.dtype(self._get_type(dtype))
msg = 'Value with data type %s is being converted to data type %s as specified.' % (g.name, s.name)
with self.assertWarnsWith(UserWarning, msg):
ret = ObjectMapper.convert_dtype(spec, value)
self.assertTupleEqual(ret, match)
self.assertIs(ret[0].dtype.type, match[1])

def test_no_spec(self):
spec_type = None
Expand Down