Split pytest into two runs given Travis memory constraints

If pytest runs all tests at once then it exceeds the alloted memory for
it by Travis CI. To deal with this, tests/test_notebooks is run by itself
in a second run of pytest. This is a bit strange, but seems to work.

As a result of this and the order that pyest runs the tests, the
docstring tests finish just before the import tests run. This means that
the last backend tested is still in memory, and so the NumPy backend
needs to get set as that is what is needed for the test_import.py

.travis.yml

@@ -10,7 +10,10 @@ install:
   - pip install --ignore-installed -U -q -e .[tensorflow,torch,mxnet,minuit,develop]  # Ensure right version of NumPy installed
 script:
   - pyflakes pyhf
-  - pytest --ignore tests/benchmarks/
+  # pytest takes up too much memory in Travis if run all at once, so split off
+  # test_notebooks.py into its own pytest run
+  - pytest --ignore tests/benchmarks/ --ignore tests/test_notebooks.py
+  - pytest tests/test_notebooks.py
 after_success: coveralls
 
 # always test (on both 'push' and 'pr' builds in Travis)

tests/test_import.py

@@ -1,12 +1,15 @@
 import pyhf
+from pyhf.tensor.numpy_backend import numpy_backend
 import pyhf.readxml
 import json
 import pytest
 import numpy as np
 
+
 def test_import_prepHistFactory():
+    pyhf.set_backend(numpy_backend())
     parsed_xml = pyhf.readxml.parse('validation/xmlimport_input/config/example.xml',
-                              'validation/xmlimport_input/')
+                                    'validation/xmlimport_input/')
 
     # build the spec, strictly checks properties included
     spec = {'channels': parsed_xml['channels']}
@@ -16,30 +19,31 @@ def test_import_prepHistFactory():
             in parsed_xml['data'][k['name']]] + pdf.config.auxdata
 
     channels = {channel['name'] for channel in pdf.spec['channels']}
-    samples = {channel['name']: [sample['name'] for sample in channel['samples']] for channel in pdf.spec['channels']}
-
+    samples = {channel['name']: [sample['name']
+                                 for sample in channel['samples']] for channel in pdf.spec['channels']}
 
     ###
-    ### signal overallsys
-    ### bkg1 overallsys (stat ignored)
-    ### bkg2 stateror (2 bins)
-    ### bkg2 overallsys
+    # signal overallsys
+    # bkg1 overallsys (stat ignored)
+    # bkg2 stateror (2 bins)
+    # bkg2 overallsys
 
     assert 'channel1' in channels
     assert 'signal' in samples['channel1']
     assert 'background1' in samples['channel1']
     assert 'background2' in samples['channel1']
 
     assert pdf.spec['channels'][0]['samples'][2]['modifiers'][0]['type'] == 'staterror'
-    assert pdf.spec['channels'][0]['samples'][2]['modifiers'][0]['data'] == [0,10.]
+    assert pdf.spec['channels'][0]['samples'][2]['modifiers'][0]['data'] == [0, 10.]
 
     assert pdf.spec['channels'][0]['samples'][1]['modifiers'][0]['type'] == 'staterror'
-    assert all(np.isclose(pdf.spec['channels'][0]['samples'][1]['modifiers'][0]['data'],[5.0, 0.0]))
+    assert all(np.isclose(
+        pdf.spec['channels'][0]['samples'][1]['modifiers'][0]['data'], [5.0, 0.0]))
 
     assert pdf.expected_actualdata(
         pdf.config.suggested_init()).tolist() == [120.0, 110.0]
 
-    assert pdf.config.auxdata_order ==  ['syst1', 'staterror_channel1', 'syst2', 'syst3']
+    assert pdf.config.auxdata_order == ['syst1', 'staterror_channel1', 'syst2', 'syst3']
 
     assert data == [122.0, 112.0, 0.0, 1.0, 1.0, 0.0, 0.0]
 
@@ -48,9 +52,11 @@ def test_import_prepHistFactory():
     assert pdf.expected_data(
         pars, include_auxdata=False).tolist() == [140, 120]
 
+
 def test_import_histosys():
+    pyhf.set_backend(numpy_backend())
     parsed_xml = pyhf.readxml.parse('validation/xmlimport_input2/config/example.xml',
-                              'validation/xmlimport_input2')
+                                    'validation/xmlimport_input2')
 
     # build the spec, strictly checks properties included
     spec = {'channels': parsed_xml['channels']}
@@ -59,7 +65,8 @@ def test_import_histosys():
     data = [binvalue for k in pdf.spec['channels'] for binvalue
             in parsed_xml['data'][k['name']]] + pdf.config.auxdata
 
-    channels = {channel['name']:channel for channel in pdf.spec['channels']}
-    samples = {channel['name']: [sample['name'] for sample in channel['samples']] for channel in pdf.spec['channels']}
+    channels = {channel['name']: channel for channel in pdf.spec['channels']}
+    samples = {channel['name']: [sample['name']
+                                 for sample in channel['samples']] for channel in pdf.spec['channels']}
 
     assert channels['channel2']['samples'][0]['modifiers'][0]['type'] == 'histosys'