PR: Make value_to_display more robust

spyder/widgets/variableexplorer/tests/test_utils.py

@@ -7,12 +7,31 @@
 Tests for utils.py
 """
 
+from collections import defaultdict
+
 # Third party imports
+import numpy as np
+import pandas as pd
 import pytest
 
 # Local imports
 from spyder.config.base import get_supported_types
-from spyder.widgets.variableexplorer.utils import sort_against, is_supported
+from spyder.widgets.variableexplorer.utils import (sort_against,
+    is_supported, value_to_display)
+
+
+def generate_complex_object():
+    """Taken from issue #4221."""
+    bug = defaultdict(list)
+    for i in range(50000):
+        a = {j:np.random.rand(10) for j in range(10)}
+        bug[i] = a
+    return bug
+
+
+COMPLEX_OBJECT = generate_complex_object()
+DF = pd.DataFrame([1,2,3])
+PANEL = pd.Panel({0: pd.DataFrame([1,2]), 1:pd.DataFrame([3,4])})
 
 
 # --- Tests
@@ -32,7 +51,7 @@ def test_sort_against_is_stable():
 
 
 def test_none_values_are_supported():
-    # None values should be displayed by default
+    """Tests that None values are displayed by default"""
     supported_types = get_supported_types()
     mode = 'editable'
     none_var = None
@@ -45,5 +64,90 @@ def test_none_values_are_supported():
     assert is_supported(none_tuple, filters=tuple(supported_types[mode]))
 
 
+def test_default_display():
+    """Tests for default_display."""
+    # Display of defaultdict
+    assert (value_to_display(COMPLEX_OBJECT) ==
+            'defaultdict object of collections module')
+
+    # Display of array of COMPLEX_OBJECT
+    assert (value_to_display(np.array(COMPLEX_OBJECT)) ==
+            'ndarray object of numpy module')
+
+    # Display of Panel
+    assert (value_to_display(PANEL) ==
+            'Panel object of pandas.core.panel module')
+
+
+def test_list_display():
+    """Tests for display of lists."""
+    long_list = list(range(100))
+
+    # Simple list
+    assert value_to_display([1, 2, 3]) == '[1, 2, 3]'
+
+    # Long list
+    assert (value_to_display(long_list) ==
+            '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, ...]')
+
+    # Short list of lists
+    assert (value_to_display([long_list] * 3) ==
+            '[[0, 1, 2, 3, 4, ...], [0, 1, 2, 3, 4, ...], [0, 1, 2, 3, 4, ...]]')
+
+    # Long list of lists
+    result = '[' + ''.join('[0, 1, 2, 3, 4, ...], '*10)[:-2] + ']'
+    assert value_to_display([long_list] * 10) == result[:70] + ' ...'
+
+    # Multiple level lists
+    assert (value_to_display([[1, 2, 3, [4], 5]] + long_list) ==
+            '[[1, 2, 3, [...], 5], 0, 1, 2, 3, 4, 5, 6, 7, 8, ...]')
+    assert value_to_display([1, 2, [DF]]) == '[1, 2, [Dataframe]]'
+    assert value_to_display([1, 2, [[DF], PANEL]]) == '[1, 2, [[...], Panel]]'
+
+    # List of complex object
+    assert value_to_display([COMPLEX_OBJECT]) == '[defaultdict]'
+
+    # List of composed objects
+    li = [COMPLEX_OBJECT, PANEL, 1, {1:2, 3:4}, DF]
+    result = '[defaultdict, Panel, 1, {1:2, 3:4}, Dataframe]'
+    assert value_to_display(li) == result
+
+
+def test_dict_display():
+    """Tests for display of dicts."""
+    long_list = list(range(100))
+    long_dict = dict(zip(list(range(100)), list(range(100))))
+
+    # Simple dict
+    assert value_to_display({0:0, 'a':'b'}) == "{0:0, 'a':'b'}"
+
+    # Long dict
+    assert (value_to_display(long_dict) ==
+            '{0:0, 1:1, 2:2, 3:3, 4:4, 5:5, 6:6, 7:7, 8:8, 9:9, ...}')
+
+    # Short list of lists
+    assert (value_to_display({1:long_dict, 2:long_dict}) ==
+            '{1:{0:0, 1:1, 2:2, 3:3, 4:4, ...}, 2:{0:0, 1:1, 2:2, 3:3, 4:4, ...}}')
+
+    # Long dict of dicts
+    result = ('{(0, 0, 0, 0, 0, ...):[0, 1, 2, 3, 4, ...], '
+               '(1, 1, 1, 1, 1, ...):[0, 1, 2, 3, 4, ...]}')
+    assert value_to_display({(0,)*100:long_list, (1,)*100:long_list}) == result[:70] + ' ...'
+
+    # Multiple level dicts
+    assert (value_to_display({0: {1:1, 2:2, 3:3, 4:{0:0}, 5:5}, 1:1}) ==
+            '{0:{1:1, 2:2, 3:3, 4:{...}, 5:5}, 1:1}')
+    assert value_to_display({0:0, 1:1, 2:2, 3:DF}) == '{0:0, 1:1, 2:2, 3:Dataframe}'
+    assert value_to_display({0:0, 1:1, 2:[[DF], PANEL]}) == '{0:0, 1:1, 2:[[...], Panel]}'
+
+    # Dict of complex object
+    assert value_to_display({0:COMPLEX_OBJECT}) == '{0:defaultdict}'
+
+    # Dict of composed objects
+    li = {0:COMPLEX_OBJECT, 1:PANEL, 2:2, 3:{0:0, 1:1}, 4:DF}
+    result = '{0:defaultdict, 1:Panel, 2:2, 3:{0:0, 1:1}, 4:Dataframe}'
+    assert value_to_display(li) == result
+
+
 if __name__ == "__main__":
     pytest.main()

spyder/widgets/variableexplorer/utils.py

@@ -5,18 +5,19 @@
 # (see spyder/__init__.py for details)
 
 """
-Utilities for the Collections editor widget and dialog
+Utilities
 """
 
 from __future__ import print_function
 
+from itertools import islice
 import re
 
 # Local imports
 from spyder.config.base import get_supported_types
 from spyder.py3compat import (NUMERIC_TYPES, TEXT_TYPES, to_text_string,
-                              is_text_string, is_binary_string, reprlib,
-                              PY2, to_binary_string)
+                              is_text_string, is_binary_string, PY2,
+                              to_binary_string, iteritems)
 from spyder.utils import programs
 from spyder import dependencies
 from spyder.config.base import _
@@ -152,17 +153,6 @@ def get_object_attrs(obj):
     return attrs
 
 
-# =============================================================================
-# Set limits for the amount of elements in the repr of collections (lists,
-# dicts, tuples and sets) and Numpy arrays
-# =============================================================================
-CollectionsRepr = reprlib.Repr()
-CollectionsRepr.maxlist = 10
-CollectionsRepr.maxdict = 10
-CollectionsRepr.maxtuple = 10
-CollectionsRepr.maxset = 10
-
-
 #==============================================================================
 # Date and datetime objects support
 #==============================================================================
@@ -256,10 +246,71 @@ def unsorted_unique(lista):
 #==============================================================================
 # Display <--> Value
 #==============================================================================
-def value_to_display(value, minmax=False):
+def default_display(value, with_module=True):
+    """Default display for unknown objects."""
+    object_type = type(value)
+    try:
+        name = object_type.__name__
+        module = object_type.__module__
+        if with_module:
+            return name + ' object of ' + module + ' module'
+        else:
+            return name
+    except:
+        type_str = to_text_string(object_type)
+        return type_str[1:-1]
+
+
+def collections_display(value, level):
+    """Display for collections (i.e. list, tuple and dict)."""
+    is_dict = isinstance(value, dict)
+
+    # Get elements
+    if is_dict:
+        elements = iteritems(value)
+    else:
+        elements = value
+
+    # Truncate values
+    truncate = False
+    if level == 1 and len(value) > 10:
+        elements = islice(elements, 10) if is_dict else value[:10]
+        truncate = True
+    elif level == 2 and len(value) > 5:
+        elements = islice(elements, 5) if is_dict else value[:5]
+        truncate = True
+
+    # Get display of each element
+    if level <= 2:
+        if is_dict:
+            displays = [value_to_display(k, level=level) + ':' +
+                        value_to_display(v, level=level)
+                        for (k, v) in list(elements)]
+        else:
+            displays = [value_to_display(e, level=level)
+                        for e in elements]
+        if truncate:
+            displays.append('...')
+        display = ', '.join(displays)
+    else:
+        display = '...'
+
+    # Return display
+    if is_dict:
+        display = '{' + display + '}'
+    elif isinstance(value, list):
+        display = '[' + display + ']'
+    else:
+        display = '(' + display + ')'
+
+    return display
+
+
+def value_to_display(value, minmax=False, level=0):
     """Convert value for display purpose"""
     # To save current Numpy threshold
     np_threshold = FakeObject
+
     try:
         numeric_numpy_types = (int64, int32, float64, float32,
                                complex128, complex64)
@@ -270,68 +321,93 @@ def value_to_display(value, minmax=False):
             # in our display
             set_printoptions(threshold=10)
         if isinstance(value, recarray):
-            fields = value.names
-            display = 'Field names: ' + ', '.join(fields)
+            if level == 0:
+                fields = value.names
+                display = 'Field names: ' + ', '.join(fields)
+            else:
+                display = 'Recarray'
         elif isinstance(value, MaskedArray):
             display = 'Masked array'
         elif isinstance(value, ndarray):
-            if minmax:
-                try:
-                    display = 'Min: %r\nMax: %r' % (value.min(), value.max())
-                except (TypeError, ValueError):
+            if level == 0:
+                if minmax:
+                    try:
+                        display = 'Min: %r\nMax: %r' % (value.min(), value.max())
+                    except (TypeError, ValueError):
+                        if value.dtype.type in numeric_numpy_types:
+                            display = repr(value)
+                        else:
+                            display = default_display(value)
+                elif value.dtype.type in numeric_numpy_types:
                     display = repr(value)
+                else:
+                    display = default_display(value)
             else:
-                display = repr(value)
-        elif isinstance(value, (list, tuple, dict, set)):
-            display = CollectionsRepr.repr(value)
+                display = 'Numpy array'
+        elif any([type(value) == t for t in [list, tuple, dict]]):
+            display = collections_display(value, level+1)
         elif isinstance(value, Image):
-            display = '%s  Mode: %s' % (address(value), value.mode)
+            if level == 0:
+                display = '%s  Mode: %s' % (address(value), value.mode)
+            else:
+                display = 'Image'
         elif isinstance(value, DataFrame):
-            cols = value.columns
-            if PY2 and len(cols) > 0:
-                # Get rid of possible BOM utf-8 data present at the
-                # beginning of a file, which gets attached to the first
-                # column header when headers are present in the first
-                # row.
-                # Fixes Issue 2514
-                try:
-                    ini_col = to_text_string(cols[0], encoding='utf-8-sig')
-                except:
-                    ini_col = to_text_string(cols[0])
-                cols = [ini_col] + [to_text_string(c) for c in cols[1:]]
+            if level == 0:
+                cols = value.columns
+                if PY2 and len(cols) > 0:
+                    # Get rid of possible BOM utf-8 data present at the
+                    # beginning of a file, which gets attached to the first
+                    # column header when headers are present in the first
+                    # row.
+                    # Fixes Issue 2514
+                    try:
+                        ini_col = to_text_string(cols[0], encoding='utf-8-sig')
+                    except:
+                        ini_col = to_text_string(cols[0])
+                    cols = [ini_col] + [to_text_string(c) for c in cols[1:]]
+                else:
+                    cols = [to_text_string(c) for c in cols]
+                display = 'Column names: ' + ', '.join(list(cols))
             else:
-                cols = [to_text_string(c) for c in cols]
-            display = 'Column names: ' + ', '.join(list(cols))
+                display = 'Dataframe'
         elif isinstance(value, NavigableString):
             # Fixes Issue 2448
             display = to_text_string(value)
+            if level > 0:
+                display = u"'" + display + u"'"
         elif isinstance(value, DatetimeIndex):
-            display = value.summary()
+            if level == 0:
+                display = value.summary()
+            else:
+                display = 'DatetimeIndex'
         elif is_binary_string(value):
             try:
                 display = to_text_string(value, 'utf8')
             except:
                 display = value
+            if level > 0:
+                display = (to_binary_string("'") + display +
+                           to_binary_string("'"))
         elif is_text_string(value):
             display = value
-        elif isinstance(value, NUMERIC_TYPES) or isinstance(value, bool) or \
-          isinstance(value, datetime.date) or \
-          isinstance(value, numeric_numpy_types):
+            if level > 0:
+                display = u"'" + display + u"'"
+        elif (isinstance(value, NUMERIC_TYPES) or isinstance(value, bool) or
+              isinstance(value, datetime.date) or
+              isinstance(value, numeric_numpy_types)):
             display = repr(value)
         else:
-            # Note: Don't trust on repr's. They can be inefficient and
-            # so freeze Spyder quite easily
-            # display = repr(value)
-            type_str = to_text_string(type(value))
-            display = type_str[1:-1]
+            if level == 0:
+                display = default_display(value)
+            else:
+                display = default_display(value, with_module=False)
     except:
-        type_str = to_text_string(type(value))
-        display = type_str[1:-1]
+        display = default_display(value)
 
-    # Truncate display at 80 chars to avoid freezing Spyder
+    # Truncate display at 70 chars to avoid freezing Spyder
     # because of large displays
-    if len(display) > 80:
-        display = display[:80].rstrip() + ' ...'
+    if len(display) > 70:
+        display = display[:70].rstrip() + ' ...'
 
     # Restore Numpy threshold
     if np_threshold is not FakeObject:
@@ -340,7 +416,6 @@ def value_to_display(value, minmax=False):
     return display
 
 
-
 def display_to_value(value, default_value, ignore_errors=True):
     """Convert back to value"""
     from qtpy.compat import from_qvariant