From 02b4d30a5527ab12bc02f55e134182686a86bea4 Mon Sep 17 00:00:00 2001
From: Matthew Feickert <matthew.feickert@cern.ch>
Date: Tue, 18 Sep 2018 14:54:02 +0200
Subject: [PATCH] Split pytest into two runs given Travis memory constraints

If pytest runs all tests at once then it exceeds the alloted memory for
it by Travis CI. To deal with this, tests/test_notebooks is run by itself
in a second run of pytest. This is a bit strange, but seems to work.

As a result of this and the order that pyest runs the tests, the
docstring tests finish just before the import tests run. This means that
the last backend tested is still in memory, and so the NumPy backend
needs to get set as that is what is needed for the test_import.py
---
 .travis.yml          |  5 ++++-
 tests/test_import.py | 33 ++++++++++++++++++++-------------
 2 files changed, 24 insertions(+), 14 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index ac277bd525..ff44a1225a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -10,7 +10,10 @@ install:
   - pip install --ignore-installed -U -q -e .[tensorflow,torch,mxnet,minuit,develop]  # Ensure right version of NumPy installed
 script:
   - pyflakes pyhf
-  - pytest --ignore tests/benchmarks/
+  # pytest takes up too much memory in Travis if run all at once, so split off
+  # test_notebooks.py into its own pytest run
+  - pytest --ignore tests/benchmarks/ --ignore tests/test_notebooks.py
+  - pytest tests/test_notebooks.py
 after_success: coveralls
 
 # always test (on both 'push' and 'pr' builds in Travis)
diff --git a/tests/test_import.py b/tests/test_import.py
index 9a820cc4e7..0f40d3c11c 100644
--- a/tests/test_import.py
+++ b/tests/test_import.py
@@ -1,12 +1,15 @@
 import pyhf
+from pyhf.tensor.numpy_backend import numpy_backend
 import pyhf.readxml
 import json
 import pytest
 import numpy as np
 
+
 def test_import_prepHistFactory():
+    pyhf.set_backend(numpy_backend())
     parsed_xml = pyhf.readxml.parse('validation/xmlimport_input/config/example.xml',
-                              'validation/xmlimport_input/')
+                                    'validation/xmlimport_input/')
 
     # build the spec, strictly checks properties included
     spec = {'channels': parsed_xml['channels']}
@@ -16,14 +19,14 @@ def test_import_prepHistFactory():
             in parsed_xml['data'][k['name']]] + pdf.config.auxdata
 
     channels = {channel['name'] for channel in pdf.spec['channels']}
-    samples = {channel['name']: [sample['name'] for sample in channel['samples']] for channel in pdf.spec['channels']}
-
+    samples = {channel['name']: [sample['name']
+                                 for sample in channel['samples']] for channel in pdf.spec['channels']}
 
     ###
-    ### signal overallsys
-    ### bkg1 overallsys (stat ignored)
-    ### bkg2 stateror (2 bins)
-    ### bkg2 overallsys
+    # signal overallsys
+    # bkg1 overallsys (stat ignored)
+    # bkg2 stateror (2 bins)
+    # bkg2 overallsys
 
     assert 'channel1' in channels
     assert 'signal' in samples['channel1']
@@ -31,15 +34,16 @@ def test_import_prepHistFactory():
     assert 'background2' in samples['channel1']
 
     assert pdf.spec['channels'][0]['samples'][2]['modifiers'][0]['type'] == 'staterror'
-    assert pdf.spec['channels'][0]['samples'][2]['modifiers'][0]['data'] == [0,10.]
+    assert pdf.spec['channels'][0]['samples'][2]['modifiers'][0]['data'] == [0, 10.]
 
     assert pdf.spec['channels'][0]['samples'][1]['modifiers'][0]['type'] == 'staterror'
-    assert all(np.isclose(pdf.spec['channels'][0]['samples'][1]['modifiers'][0]['data'],[5.0, 0.0]))
+    assert all(np.isclose(
+        pdf.spec['channels'][0]['samples'][1]['modifiers'][0]['data'], [5.0, 0.0]))
 
     assert pdf.expected_actualdata(
         pdf.config.suggested_init()).tolist() == [120.0, 110.0]
 
-    assert pdf.config.auxdata_order ==  ['syst1', 'staterror_channel1', 'syst2', 'syst3']
+    assert pdf.config.auxdata_order == ['syst1', 'staterror_channel1', 'syst2', 'syst3']
 
     assert data == [122.0, 112.0, 0.0, 1.0, 1.0, 0.0, 0.0]
 
@@ -48,9 +52,11 @@ def test_import_prepHistFactory():
     assert pdf.expected_data(
         pars, include_auxdata=False).tolist() == [140, 120]
 
+
 def test_import_histosys():
+    pyhf.set_backend(numpy_backend())
     parsed_xml = pyhf.readxml.parse('validation/xmlimport_input2/config/example.xml',
-                              'validation/xmlimport_input2')
+                                    'validation/xmlimport_input2')
 
     # build the spec, strictly checks properties included
     spec = {'channels': parsed_xml['channels']}
@@ -59,7 +65,8 @@ def test_import_histosys():
     data = [binvalue for k in pdf.spec['channels'] for binvalue
             in parsed_xml['data'][k['name']]] + pdf.config.auxdata
 
-    channels = {channel['name']:channel for channel in pdf.spec['channels']}
-    samples = {channel['name']: [sample['name'] for sample in channel['samples']] for channel in pdf.spec['channels']}
+    channels = {channel['name']: channel for channel in pdf.spec['channels']}
+    samples = {channel['name']: [sample['name']
+                                 for sample in channel['samples']] for channel in pdf.spec['channels']}
 
     assert channels['channel2']['samples'][0]['modifiers'][0]['type'] == 'histosys'