Unicode support

cf_units/__init__.py

@@ -809,6 +809,10 @@ def __init__(self, unit, calendar=None):
         else:
             unit = str(unit).strip()
 
+        if six.PY2:
+            # Ensure that the string is a unicode object.
+            unit = six.text_type(unit.decode('utf8'))
+
         if unit.lower().endswith(' utc'):
             unit = unit[:unit.lower().rfind(' utc')]
 
@@ -831,7 +835,7 @@ def __init__(self, unit, calendar=None):
         else:
             category = _CATEGORY_UDUNIT
             try:
-                ut_unit = _ud.parse(_ud_system, unit.encode('ascii'), UT_ASCII)
+                ut_unit = _ud.parse(_ud_system, unit.encode('utf-8'), UT_UTF8)
             except _ud.UdunitsError as e:
                 self._propogate_error('Failed to parse unit "%s"' % unit, e)
             if _OP_SINCE in unit.lower():
@@ -1893,7 +1897,7 @@ def utime(self):
         if self.is_long_time_interval():
             interval = self.origin.split(' ')[0]
             emsg = ('Time units with interval of "months", "years" '
-                    '(or singular of these) cannot be processed, got {!r}.')
+                    '(or singular of these) cannot be processed, got "{!s}".')
             raise ValueError(emsg.format(interval))
 
         #

cf_units/tests/integration/test_date2num.py

@@ -1,4 +1,4 @@
-# (C) British Crown Copyright 2016 - 2018, Met Office
+# (C) British Crown Copyright 2016 - 2019, Met Office
 #
 # This file is part of cf-units.
 #
@@ -73,7 +73,7 @@ def test_long_time_interval(self):
         # This test should fail with an error that we need to catch properly.
         unit = 'years since 1970-01-01'
         date = datetime.datetime(1970, 1, 1, 0, 0, 5)
-        exp_emsg = 'interval of "months", "years" .* got \'years\'.'
+        exp_emsg = 'interval of "months", "years" .* got "years".'
         with six.assertRaisesRegex(self, ValueError, exp_emsg):
             date2num(date, unit, self.calendar)
 

cf_units/tests/test_coding_standards.py

@@ -1,4 +1,4 @@
-# (C) British Crown Copyright 2013 - 2018, Met Office
+# (C) British Crown Copyright 2013 - 2019, Met Office
 #
 # This file is part of cf-units.
 #
@@ -52,9 +52,9 @@
 
 
 LICENSE_RE_PATTERN = re.escape(LICENSE_TEMPLATE).replace(r'\{YEARS\}', '(.*?)')
-# Add shebang possibility to the LICENSE_RE_PATTERN
-LICENSE_RE_PATTERN = r'(\#\!.*\n)?' + LICENSE_RE_PATTERN
-LICENSE_RE = re.compile(LICENSE_RE_PATTERN, re.MULTILINE)
+SHEBANG = r'(\#\!.*\n)?'
+ENCODING = r'(\# \-\*\- coding\: .* \-\*\-\n)?'
+LICENSE_RE = re.compile(SHEBANG + ENCODING + LICENSE_RE_PATTERN, re.MULTILINE)
 
 
 # Guess cf_units repo directory of cf_units - realpath is used to mitigate

cf_units/tests/test_unit.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # (C) British Crown Copyright 2010 - 2019, Met Office
 #
 # This file is part of cf-units.
@@ -74,6 +75,17 @@ def test_unsupported_calendar(self):
         with six.assertRaisesRegex(self, ValueError, 'unsupported calendar'):
             Unit('hours since 1970-01-01 00:00:00', calendar='wibble')
 
+    def test_unicode_valid(self):
+        # Some unicode characters are allowed.
+        u = Unit('m²')
+        assert u.symbol == 'm2'
+
+    def test_unicode_invalid(self):
+        # Not all unicode characters are allowed.
+        msg = '[UT_UNKNOWN] Failed to parse unit "ø"'
+        with self.assertRaises(ValueError, msg=msg):
+            Unit('ø')
+
 
 class Test_modulus(unittest.TestCase):