From e63781d5f8af5f6c580444a258dcd27fa0c6c8ac Mon Sep 17 00:00:00 2001 From: Lukas Heinrich Date: Sun, 29 Dec 2019 17:30:38 +0100 Subject: [PATCH 1/6] change qmu API and remove test_stat return --- .../binderexample/StatisticalAnalysis.ipynb | 81 +++++++++---------- docs/examples/notebooks/hello-world.ipynb | 41 ++++------ src/pyhf/infer/__init__.py | 13 +-- src/pyhf/infer/test_statistics.py | 6 +- src/pyhf/infer/utils.py | 4 +- tests/benchmarks/test_benchmark.py | 1 - tests/test_backend_consistency.py | 5 +- tests/test_infer.py | 25 ------ tests/test_public_api.py | 1 - 9 files changed, 60 insertions(+), 117 deletions(-) diff --git a/docs/examples/notebooks/binderexample/StatisticalAnalysis.ipynb b/docs/examples/notebooks/binderexample/StatisticalAnalysis.ipynb index aac70b0468..568a050c49 100644 --- a/docs/examples/notebooks/binderexample/StatisticalAnalysis.ipynb +++ b/docs/examples/notebooks/binderexample/StatisticalAnalysis.ipynb @@ -116,7 +116,11 @@ " 'auxdata': [1.0],\n", " 'bounds': [[0.5, 1.5]],\n", " 'inits': [1.0],\n", - " 'sigmas': [0.1]}]}}" + " 'sigmas': [0.1]},\n", + " {'name': 'SigXsecOverSM',\n", + " 'bounds': [[0.0, 3.0]],\n", + " 'inits': [1.0],\n", + " 'fixed': False}]}}" ] }, "execution_count": 5, @@ -799,7 +803,7 @@ "};\n", "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Pan axes with left mouse, zoom with right\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n", "\n", - "mpl.extensions = [\"eps\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\"];\n", + "mpl.extensions = [\"eps\", \"jpeg\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n", "\n", "mpl.default_extension = \"png\";var comm_websocket_adapter = function(comm) {\n", " // Create a \"websocket\"-like object which calls the given IPython comm\n", @@ -1020,7 +1024,7 @@ { "data": { "text/html": [ - "" + "" ], "text/plain": [ "" @@ -1032,12 +1036,12 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "d1ec424697f246eabfd1410575913c16", + "model_id": "4479d7d0f95e489dbe4527908c457879", "version_major": 2, "version_minor": 0 }, "text/plain": [ - "interactive(children=(FloatSlider(value=1.0, description='lumi', max=1.5, min=0.5), IntSlider(value=5, descrip…" + "interactive(children=(FloatSlider(value=1.0, description='lumi', max=1.5, min=0.5), FloatSlider(value=1.5, des…" ] }, "metadata": {}, @@ -1063,7 +1067,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/Users/jovyan/pyhf/src/pyhf/tensor/numpy_backend.py:252: RuntimeWarning: invalid value encountered in log\n", + "/Users/lukasheinrich/Code/pyhfdev/dev/pyhfsrc/src/pyhf/tensor/numpy_backend.py:253: RuntimeWarning: invalid value encountered in log\n", " return n * np.log(lam) - lam - gammaln(n + 1.0)\n" ] } @@ -1648,7 +1652,7 @@ "};\n", "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Pan axes with left mouse, zoom with right\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n", "\n", - "mpl.extensions = [\"eps\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\"];\n", + "mpl.extensions = [\"eps\", \"jpeg\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n", "\n", "mpl.default_extension = \"png\";var comm_websocket_adapter = function(comm) {\n", " // Create a \"websocket\"-like object which calls the given IPython comm\n", @@ -1869,7 +1873,7 @@ { "data": { "text/html": [ - "" + "" ], "text/plain": [ "" @@ -1910,23 +1914,9 @@ "metadata": {}, "outputs": [], "source": [ - "def plot_results(ax, test_mus, cls_obs, cls_exp, test_size=0.05):\n", - " ax.plot(mu_tests, cls_obs, c = 'k')\n", - " for i,c in zip(range(5),['k','k','k','k','k']):\n", - " ax.plot(mu_tests, cls_exp[i], c = c, linestyle = 'dotted' if i!=2 else 'dashed')\n", - " ax.fill_between(test_mus,cls_exp[0],cls_exp[-1], facecolor = 'y')\n", - " ax.fill_between(test_mus,cls_exp[1],cls_exp[-2], facecolor = 'g')\n", - " ax.plot(test_mus,[test_size]*len(test_mus), c = 'r')\n", - " ax.set_ylim(0,1)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "def invert_interval(test_mus, cls_obs, cls_exp, test_size=0.05):\n", + "def invert_interval(test_mus, hypo_tests, test_size=0.05):\n", + " cls_obs = np.array([test[0] for test in hypo_tests]).flatten()\n", + " cls_exp = [np.array([test[1][i] for test in hypo_tests]).flatten() for i in range(5)]\n", " crossing_test_stats = {'exp': [], 'obs': None}\n", " for cls_exp_sigma in cls_exp:\n", " crossing_test_stats['exp'].append(\n", @@ -1942,7 +1932,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 12, "metadata": { "scrolled": false }, @@ -2506,7 +2496,7 @@ "};\n", "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Pan axes with left mouse, zoom with right\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n", "\n", - "mpl.extensions = [\"eps\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\"];\n", + "mpl.extensions = [\"eps\", \"jpeg\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n", "\n", "mpl.default_extension = \"png\";var comm_websocket_adapter = function(comm) {\n", " // Create a \"websocket\"-like object which calls the given IPython comm\n", @@ -2727,7 +2717,7 @@ { "data": { "text/html": [ - "" + "" ], "text/plain": [ "" @@ -2740,30 +2730,24 @@ "source": [ "mu_tests = np.linspace(0, 1, 16)\n", "hypo_tests = [pyhf.infer.hypotest(mu, data, pdf, pdf.config.suggested_init(), pdf.config.suggested_bounds(),\n", - " return_expected_set=True, return_test_statistics=True)\n", + " return_expected_set=True)\n", " for mu in mu_tests]\n", "\n", - "test_stats = np.array([test[-1][0] for test in hypo_tests]).flatten()\n", - "cls_obs = np.array([test[0] for test in hypo_tests]).flatten()\n", - "cls_exp = [np.array([test[1][i] for test in hypo_tests]).flatten() for i in range(5)]\n", "\n", - "fig, (ax1,ax2) = plt.subplots(1, 2)\n", - "fig.set_size_inches(15, 5)\n", + "fig, (ax1) = plt.subplots(1, 1)\n", + "fig.set_size_inches(7, 5)\n", "\n", "ax1.set_title(u'Hypothesis Tests')\n", "ax1.set_ylabel(u'CLs')\n", "ax1.set_xlabel(u'µ')\n", - "plot_results(ax1, mu_tests, cls_obs, cls_exp)\n", - "\n", "\n", - "ax2.set_title(u'Test Statistic')\n", - "ax2.set_xlabel(u'µ')\n", - "ax2.plot(mu_tests,test_stats);" + "import pyhf.contrib.viz.brazil\n", + "pyhf.contrib.viz.brazil.plot_results(ax1, mu_tests, hypo_tests)\n" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 13, "metadata": {}, "outputs": [ { @@ -2781,20 +2765,27 @@ } ], "source": [ - "results = invert_interval(mu_tests, cls_obs, cls_exp)\n", + "results = invert_interval(mu_tests, hypo_tests)\n", "\n", "print('Observed Limit: {:.2f}'.format(results['obs']))\n", "print('-----')\n", "for i, n_sigma in enumerate(np.arange(-2,3)):\n", " print('Expected Limit{}: {:.3f}'.format('' if n_sigma==0 else '({} σ)'.format(n_sigma),results['exp'][i]))" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { "kernelspec": { - "display_name": "itkdb", + "display_name": "Python 3", "language": "python", - "name": "itkdb" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -2806,7 +2797,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.3" + "version": "3.6.6" }, "widgets": { "application/vnd.jupyter.widget-state+json": { @@ -9050,4 +9041,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/docs/examples/notebooks/hello-world.ipynb b/docs/examples/notebooks/hello-world.ipynb index b51a9b30e9..8857f44504 100644 --- a/docs/examples/notebooks/hello-world.ipynb +++ b/docs/examples/notebooks/hello-world.ipynb @@ -41,7 +41,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Observed: [0.05290116], Expected: [0.06445521]\n" + "Observed: 0.05290116224852556, Expected: 0.06445521290832805\n" ] } ], @@ -67,7 +67,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Observed CL_s: [0.05290116], CL_sb: [0.0236], CL_b: [0.44611493]\n" + "Observed CL_s: 0.05290116224852556, CL_sb: 0.023599998519978738, CL_b: 0.4461149342826869\n" ] } ], @@ -120,13 +120,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Observed CL_s: [0.05290116]\n", + "Observed CL_s: 0.05290116224852556\n", "\n", - "Expected CL_s(-2 σ): [0.00260641]\n", - "Expected CL_s(-1 σ): [0.01382066]\n", - "Expected CL_s : [0.06445521]\n", - "Expected CL_s(1 σ): [0.23526104]\n", - "Expected CL_s(2 σ): [0.57304182]\n" + "Expected CL_s(-2 σ): 0.0026064088679947964\n", + "Expected CL_s(-1 σ): 0.013820657528619273\n", + "Expected CL_s : 0.06445521290832805\n", + "Expected CL_s(1 σ): 0.23526103626937836\n", + "Expected CL_s(2 σ): 0.5730418174887743\n" ] } ], @@ -138,29 +138,18 @@ ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, - "source": [ - "**Returning the test statistics for the observed and Asimov data**" - ] + "outputs": [], + "source": [] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "q_mu: [3.93824492], Asimov q_mu: [3.41886758]\n" - ] - } - ], - "source": [ - "CLs_obs, test_statistics = pyhf.infer.hypotest(1.0, [51, 48] + pdf.config.auxdata, pdf, return_test_statistics=True)\n", - "print('q_mu: {}, Asimov q_mu: {}'.format(test_statistics[0], test_statistics[1]))" - ] + "outputs": [], + "source": [] } ], "metadata": { diff --git a/src/pyhf/infer/__init__.py b/src/pyhf/infer/__init__.py index 462eb8dd32..598e34db0a 100644 --- a/src/pyhf/infer/__init__.py +++ b/src/pyhf/infer/__init__.py @@ -27,7 +27,6 @@ def hypotest( return_tail_probs (bool): Bool for returning :math:`\textrm{CL}_{s+b}` and :math:`\textrm{CL}_{b}` return_expected (bool): Bool for returning :math:`\textrm{CL}_{\textrm{exp}}` return_expected_set (bool): Bool for returning the :math:`(-2,-1,0,1,2)\sigma` :math:`\textrm{CL}_{\textrm{exp}}` --- the "Brazil band" - return_test_statistics (bool): Bool for returning :math:`q_{\mu}` and :math:`q_{\mu,A}` Returns: Tuple of Floats and lists of Floats: @@ -74,8 +73,6 @@ def hypotest( for :math:`\mu'=0` and :math:`N \in \left\{-2, -1, 0, 1, 2\right\}`. These values define the boundaries of an uncertainty band sometimes referred to as the "Brazil band". Only returned when ``return_expected_set`` is ``True``. - - :math:`\left[q_{\mu}, q_{\mu,A}\right]`: The test statistics for the observed and Asimov datasets respectively. Only returned when ``return_test_statistics`` is ``True``. - """ init_pars = init_pars or pdf.config.suggested_init() par_bounds = par_bounds or pdf.config.suggested_bounds() @@ -84,14 +81,10 @@ def hypotest( asimov_mu = 0.0 asimov_data = generate_asimov_data(asimov_mu, data, pdf, init_pars, par_bounds) - qmu_v = tensorlib.clip( - qmu(poi_test, data, pdf, init_pars, par_bounds), 0, max_value=None - ) + qmu_v = qmu(poi_test, data, pdf, init_pars, par_bounds) sqrtqmu_v = tensorlib.sqrt(qmu_v) - qmuA_v = tensorlib.clip( - qmu(poi_test, asimov_data, pdf, init_pars, par_bounds), 0, max_value=None - ) + qmuA_v = qmu(poi_test, asimov_data, pdf, init_pars, par_bounds) sqrtqmuA_v = tensorlib.sqrt(qmuA_v) CLsb, CLb, CLs = pvals_from_teststat(sqrtqmu_v, sqrtqmuA_v, qtilde=qtilde) @@ -109,8 +102,6 @@ def hypotest( _returns.append(CLs_exp) elif kwargs.get('return_expected'): _returns.append(pvals_from_teststat_expected(sqrtqmuA_v)[-1]) - if kwargs.get('return_test_statistics'): - _returns.append([qmu_v, qmuA_v]) # Enforce a consistent return type of the observed CLs return tuple(_returns) if len(_returns) > 1 else _returns[0] diff --git a/src/pyhf/infer/test_statistics.py b/src/pyhf/infer/test_statistics.py index 6efb12cef0..e93d1d24c6 100644 --- a/src/pyhf/infer/test_statistics.py +++ b/src/pyhf/infer/test_statistics.py @@ -38,6 +38,6 @@ def qmu(mu, data, pdf, init_pars, par_bounds): ) qmu = fixed_poi_fit_lhood_val - unconstrained_fit_lhood_val qmu = tensorlib.where( - muhatbhat[pdf.config.poi_index] > mu, tensorlib.astensor([0]), qmu - ) - return qmu + muhatbhat[pdf.config.poi_index] > mu, tensorlib.astensor(0.0), qmu + )[0] + return tensorlib.clip(qmu,0,max_value = None) diff --git a/src/pyhf/infer/utils.py b/src/pyhf/infer/utils.py index 1a77763ec2..908802da60 100644 --- a/src/pyhf/infer/utils.py +++ b/src/pyhf/infer/utils.py @@ -60,7 +60,7 @@ def _false_case(): CLsb = 1 - tensorlib.normal_cdf(nullval) CLb = 1 - tensorlib.normal_cdf(altval) CLs = CLsb / CLb - return CLsb, CLb, CLs + return tensorlib.reshape(CLsb,(1,)), tensorlib.reshape(CLb,(1,)), tensorlib.reshape(CLs,(1,)) def pvals_from_teststat_expected(sqrtqmuA_v, nsigma=0): @@ -86,4 +86,4 @@ def pvals_from_teststat_expected(sqrtqmuA_v, nsigma=0): CLsb = tensorlib.normal_cdf(nsigma - sqrtqmuA_v) CLb = tensorlib.normal_cdf(nsigma) CLs = CLsb / CLb - return CLsb, CLb, CLs + return tensorlib.reshape(CLsb,(1,)), tensorlib.reshape(CLb,(1,)), tensorlib.reshape(CLs,(1,)) diff --git a/tests/benchmarks/test_benchmark.py b/tests/benchmarks/test_benchmark.py index 5d185e9cb8..48c2f2155b 100644 --- a/tests/benchmarks/test_benchmark.py +++ b/tests/benchmarks/test_benchmark.py @@ -62,7 +62,6 @@ def hypotest(pdf, data): return_tail_probs=True, return_expected=True, return_expected_set=True, - return_test_statistics=True, ) diff --git a/tests/test_backend_consistency.py b/tests/test_backend_consistency.py index 99650b8cfb..53ff85fbe2 100644 --- a/tests/test_backend_consistency.py +++ b/tests/test_backend_consistency.py @@ -117,14 +117,13 @@ def test_hypotest_q_mu( backend.session = tf.compat.v1.Session() pyhf.set_backend(backend) - q_mu = pyhf.infer.hypotest( + q_mu = pyhf.infer.test_statistics.qmu( 1.0, data, pdf, pdf.config.suggested_init(), pdf.config.suggested_bounds(), - return_test_statistics=True, - )[-1][0] + ) test_statistic.append(pyhf.tensorlib.tolist(q_mu)) # compare to NumPy/SciPy diff --git a/tests/test_infer.py b/tests/test_infer.py index 074528aae3..66f9ddb080 100644 --- a/tests/test_infer.py +++ b/tests/test_infer.py @@ -87,28 +87,3 @@ def test_hypotest_return_expected_set(tmpdir, hypotest_args): assert len(result[3]) == 5 assert check_uniform_type(result[3]) - -def test_hypotest_return_test_statistics(tmpdir, hypotest_args): - """ - Check that the return structure of pyhf.infer.hypotest with the - additon of the return_test_statistics keyword arg is as expected - """ - tb = pyhf.tensorlib - - kwargs = { - 'return_tail_probs': True, - 'return_expected': True, - 'return_expected_set': True, - 'return_test_statistics': True, - } - result = pyhf.infer.hypotest(*hypotest_args, **kwargs) - # CLs_obs, [CLsb, CLb], CLs_exp, CLs_exp @[-2, -1, 0, +1, +2]sigma, [q_mu, q_mu_Asimov] - assert len(list(result)) == 5 - assert isinstance(result[0], type(tb.astensor(result[0]))) - assert len(result[1]) == 2 - assert check_uniform_type(result[1]) - assert isinstance(result[2], type(tb.astensor(result[2]))) - assert len(result[3]) == 5 - assert check_uniform_type(result[3]) - assert len(result[4]) == 2 - assert check_uniform_type(result[4]) diff --git a/tests/test_public_api.py b/tests/test_public_api.py index ad373a6ec9..25213a0218 100644 --- a/tests/test_public_api.py +++ b/tests/test_public_api.py @@ -73,7 +73,6 @@ def test_hypotest(backend, model_setup): init_pars, model.config.suggested_bounds(), return_expected_set=True, - return_test_statistics=True, ) From 962a2ab5a46dd16c97056b72391f4d2af6690e55 Mon Sep 17 00:00:00 2001 From: Lukas Heinrich Date: Sun, 29 Dec 2019 17:33:31 +0100 Subject: [PATCH 2/6] black --- src/pyhf/infer/test_statistics.py | 2 +- src/pyhf/infer/utils.py | 12 ++++++++++-- tests/test_backend_consistency.py | 6 +----- tests/test_infer.py | 1 - 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/pyhf/infer/test_statistics.py b/src/pyhf/infer/test_statistics.py index e93d1d24c6..dcb8604c78 100644 --- a/src/pyhf/infer/test_statistics.py +++ b/src/pyhf/infer/test_statistics.py @@ -40,4 +40,4 @@ def qmu(mu, data, pdf, init_pars, par_bounds): qmu = tensorlib.where( muhatbhat[pdf.config.poi_index] > mu, tensorlib.astensor(0.0), qmu )[0] - return tensorlib.clip(qmu,0,max_value = None) + return tensorlib.clip(qmu, 0, max_value=None) diff --git a/src/pyhf/infer/utils.py b/src/pyhf/infer/utils.py index 908802da60..248243807b 100644 --- a/src/pyhf/infer/utils.py +++ b/src/pyhf/infer/utils.py @@ -60,7 +60,11 @@ def _false_case(): CLsb = 1 - tensorlib.normal_cdf(nullval) CLb = 1 - tensorlib.normal_cdf(altval) CLs = CLsb / CLb - return tensorlib.reshape(CLsb,(1,)), tensorlib.reshape(CLb,(1,)), tensorlib.reshape(CLs,(1,)) + return ( + tensorlib.reshape(CLsb, (1,)), + tensorlib.reshape(CLb, (1,)), + tensorlib.reshape(CLs, (1,)), + ) def pvals_from_teststat_expected(sqrtqmuA_v, nsigma=0): @@ -86,4 +90,8 @@ def pvals_from_teststat_expected(sqrtqmuA_v, nsigma=0): CLsb = tensorlib.normal_cdf(nsigma - sqrtqmuA_v) CLb = tensorlib.normal_cdf(nsigma) CLs = CLsb / CLb - return tensorlib.reshape(CLsb,(1,)), tensorlib.reshape(CLb,(1,)), tensorlib.reshape(CLs,(1,)) + return ( + tensorlib.reshape(CLsb, (1,)), + tensorlib.reshape(CLb, (1,)), + tensorlib.reshape(CLs, (1,)), + ) diff --git a/tests/test_backend_consistency.py b/tests/test_backend_consistency.py index 53ff85fbe2..e266d2e4e4 100644 --- a/tests/test_backend_consistency.py +++ b/tests/test_backend_consistency.py @@ -118,11 +118,7 @@ def test_hypotest_q_mu( pyhf.set_backend(backend) q_mu = pyhf.infer.test_statistics.qmu( - 1.0, - data, - pdf, - pdf.config.suggested_init(), - pdf.config.suggested_bounds(), + 1.0, data, pdf, pdf.config.suggested_init(), pdf.config.suggested_bounds(), ) test_statistic.append(pyhf.tensorlib.tolist(q_mu)) diff --git a/tests/test_infer.py b/tests/test_infer.py index 66f9ddb080..0577e14d15 100644 --- a/tests/test_infer.py +++ b/tests/test_infer.py @@ -86,4 +86,3 @@ def test_hypotest_return_expected_set(tmpdir, hypotest_args): assert isinstance(result[2], type(tb.astensor(result[2]))) assert len(result[3]) == 5 assert check_uniform_type(result[3]) - From db5ed630635f1ec9f1f5670b61cf79ad2c9f9c88 Mon Sep 17 00:00:00 2001 From: Lukas Heinrich Date: Sun, 29 Dec 2019 17:44:13 +0100 Subject: [PATCH 3/6] fix failing test --- src/pyhf/infer/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pyhf/infer/utils.py b/src/pyhf/infer/utils.py index 248243807b..9651f6970a 100644 --- a/src/pyhf/infer/utils.py +++ b/src/pyhf/infer/utils.py @@ -55,7 +55,7 @@ def _false_case(): return nullval, altval nullval, altval = tensorlib.conditional( - (sqrtqmu_v < sqrtqmuA_v)[0], _true_case, _false_case + (sqrtqmu_v < sqrtqmuA_v), _true_case, _false_case ) CLsb = 1 - tensorlib.normal_cdf(nullval) CLb = 1 - tensorlib.normal_cdf(altval) From 41477646e9d7d58ae727463faa8dfdf69a3519fe Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Sun, 29 Dec 2019 23:29:51 -0600 Subject: [PATCH 4/6] Apply Black --- .../binderexample/StatisticalAnalysis.ipynb | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/docs/examples/notebooks/binderexample/StatisticalAnalysis.ipynb b/docs/examples/notebooks/binderexample/StatisticalAnalysis.ipynb index 568a050c49..2bde1283e9 100644 --- a/docs/examples/notebooks/binderexample/StatisticalAnalysis.ipynb +++ b/docs/examples/notebooks/binderexample/StatisticalAnalysis.ipynb @@ -1067,7 +1067,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/Users/lukasheinrich/Code/pyhfdev/dev/pyhfsrc/src/pyhf/tensor/numpy_backend.py:253: RuntimeWarning: invalid value encountered in log\n", + "/Users/jovyan/pyhf/src/pyhf/tensor/numpy_backend.py:253: RuntimeWarning: invalid value encountered in log\n", " return n * np.log(lam) - lam - gammaln(n + 1.0)\n" ] } @@ -2729,20 +2729,29 @@ ], "source": [ "mu_tests = np.linspace(0, 1, 16)\n", - "hypo_tests = [pyhf.infer.hypotest(mu, data, pdf, pdf.config.suggested_init(), pdf.config.suggested_bounds(),\n", - " return_expected_set=True)\n", - " for mu in mu_tests]\n", + "hypo_tests = [\n", + " pyhf.infer.hypotest(\n", + " mu,\n", + " data,\n", + " pdf,\n", + " pdf.config.suggested_init(),\n", + " pdf.config.suggested_bounds(),\n", + " return_expected_set=True,\n", + " )\n", + " for mu in mu_tests\n", + "]\n", "\n", "\n", "fig, (ax1) = plt.subplots(1, 1)\n", "fig.set_size_inches(7, 5)\n", "\n", - "ax1.set_title(u'Hypothesis Tests')\n", - "ax1.set_ylabel(u'CLs')\n", - "ax1.set_xlabel(u'µ')\n", + "ax1.set_title(u\"Hypothesis Tests\")\n", + "ax1.set_ylabel(u\"CLs\")\n", + "ax1.set_xlabel(u\"µ\")\n", "\n", "import pyhf.contrib.viz.brazil\n", - "pyhf.contrib.viz.brazil.plot_results(ax1, mu_tests, hypo_tests)\n" + "\n", + "pyhf.contrib.viz.brazil.plot_results(ax1, mu_tests, hypo_tests);" ] }, { @@ -2772,13 +2781,6 @@ "for i, n_sigma in enumerate(np.arange(-2,3)):\n", " print('Expected Limit{}: {:.3f}'.format('' if n_sigma==0 else '({} σ)'.format(n_sigma),results['exp'][i]))" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { From c220eae4a727dcf789b20578187dac4431abb802 Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Sun, 29 Dec 2019 23:41:13 -0600 Subject: [PATCH 5/6] Apply more Black --- .../binderexample/StatisticalAnalysis.ipynb | 146 +++++++++++------- 1 file changed, 91 insertions(+), 55 deletions(-) diff --git a/docs/examples/notebooks/binderexample/StatisticalAnalysis.ipynb b/docs/examples/notebooks/binderexample/StatisticalAnalysis.ipynb index 2bde1283e9..6cd756972f 100644 --- a/docs/examples/notebooks/binderexample/StatisticalAnalysis.ipynb +++ b/docs/examples/notebooks/binderexample/StatisticalAnalysis.ipynb @@ -169,41 +169,59 @@ "metadata": {}, "outputs": [], "source": [ - "par_name_dict = {k: v['slice'].start for k,v in pdf.config.par_map.items()}\n", - "all_par_settings = {n[0]: tuple(m) for n,m in zip(sorted(reversed(list(par_name_dict.items())), key=lambda x:x[1]), pdf.config.suggested_bounds())}\n", - "default_par_settings = {n[0]: sum(tuple(m))/2.0 for n,m in all_par_settings.items()}\n", + "par_name_dict = {k: v[\"slice\"].start for k, v in pdf.config.par_map.items()}\n", + "all_par_settings = {\n", + " n[0]: tuple(m)\n", + " for n, m in zip(\n", + " sorted(reversed(list(par_name_dict.items())), key=lambda x: x[1]),\n", + " pdf.config.suggested_bounds(),\n", + " )\n", + "}\n", + "default_par_settings = {n[0]: sum(tuple(m)) / 2.0 for n, m in all_par_settings.items()}\n", + "\n", "\n", "def get_mc_counts(pars):\n", " deltas, factors = pdf._modifications(pars)\n", - " allsum = pyhf.tensorlib.concatenate(deltas + [pyhf.tensorlib.astensor(pdf.nominal_rates)])\n", - " nom_plus_delta = pyhf.tensorlib.sum(allsum,axis=0)\n", - " nom_plus_delta = pyhf.tensorlib.reshape(nom_plus_delta,(1,)+pyhf.tensorlib.shape(nom_plus_delta))\n", + " allsum = pyhf.tensorlib.concatenate(\n", + " deltas + [pyhf.tensorlib.astensor(pdf.nominal_rates)]\n", + " )\n", + " nom_plus_delta = pyhf.tensorlib.sum(allsum, axis=0)\n", + " nom_plus_delta = pyhf.tensorlib.reshape(\n", + " nom_plus_delta, (1,) + pyhf.tensorlib.shape(nom_plus_delta)\n", + " )\n", " allfac = pyhf.tensorlib.concatenate(factors + [nom_plus_delta])\n", - " return pyhf.tensorlib.product(allfac,axis=0)\n", + " return pyhf.tensorlib.product(allfac, axis=0)\n", + "\n", "\n", "animate_plot_pieces = None\n", + "\n", + "\n", "def init_plot(fig, ax, par_settings):\n", " global animate_plot_pieces\n", - " \n", + "\n", " nbins = sum(list(pdf.config.channel_nbins.values()))\n", " x = np.arange(nbins)\n", " data = np.zeros(nbins)\n", " items = []\n", " for i in [3, 2, 1, 0]:\n", " items.append(ax.bar(x, data, 1, alpha=1.0))\n", - " animate_plot_pieces = (items, ax.scatter(x, workspace.data(pdf, with_aux=False), c='k', alpha=1., zorder=99))\n", + " animate_plot_pieces = (\n", + " items,\n", + " ax.scatter(x, workspace.data(pdf, with_aux=False), c=\"k\", alpha=1.0, zorder=99),\n", + " )\n", + "\n", "\n", "def animate(ax=None, fig=None, **par_settings):\n", " global animate_plot_pieces\n", " items, obs = animate_plot_pieces\n", " pars = pyhf.tensorlib.astensor(pdf.config.suggested_init())\n", - " for k,v in par_settings.items():\n", + " for k, v in par_settings.items():\n", " pars[par_name_dict[k]] = v\n", "\n", " mc_counts = get_mc_counts(pars)\n", " rectangle_collection = zip(*map(lambda x: x.patches, items))\n", - " \n", - " for rectangles,binvalues in zip(rectangle_collection, mc_counts[:,0].T):\n", + "\n", + " for rectangles, binvalues in zip(rectangle_collection, mc_counts[:, 0].T):\n", " offset = 0\n", " for sample_index in [3, 2, 1, 0]:\n", " rect = rectangles[sample_index]\n", @@ -214,20 +232,21 @@ "\n", " fig.canvas.draw()\n", "\n", + "\n", "def plot(ax=None, order=[3, 2, 1, 0], **par_settings):\n", " pars = pyhf.tensorlib.astensor(pdf.config.suggested_init())\n", - " for k,v in par_settings.items():\n", + " for k, v in par_settings.items():\n", " pars[par_name_dict[k]] = v\n", "\n", " mc_counts = get_mc_counts(pars)\n", " bottom = None\n", " # nb: bar_data[0] because evaluating only one parset\n", - " for i,sample_index in enumerate(order):\n", + " for i, sample_index in enumerate(order):\n", " data = mc_counts[sample_index][0]\n", " x = np.arange(len(data))\n", - " ax.bar(x, data, 1, bottom = bottom, alpha = 1.0)\n", - " bottom = data if i==0 else bottom + data\n", - " ax.scatter(x, workspace.data(pdf, with_aux=False), c = 'k', alpha = 1., zorder=99)" + " ax.bar(x, data, 1, bottom=bottom, alpha=1.0)\n", + " bottom = data if i == 0 else bottom + data\n", + " ax.scatter(x, workspace.data(pdf, with_aux=False), c=\"k\", alpha=1.0, zorder=99)" ] }, { @@ -803,7 +822,7 @@ "};\n", "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Pan axes with left mouse, zoom with right\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n", "\n", - "mpl.extensions = [\"eps\", \"jpeg\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n", + "mpl.extensions = [\"eps\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\"];\n", "\n", "mpl.default_extension = \"png\";var comm_websocket_adapter = function(comm) {\n", " // Create a \"websocket\"-like object which calls the given IPython comm\n", @@ -973,9 +992,12 @@ " // Check for shift+enter\n", " if (event.shiftKey && event.which == 13) {\n", " this.canvas_div.blur();\n", - " // select the cell after this one\n", - " var index = IPython.notebook.find_cell_index(this.cell_info[0]);\n", - " IPython.notebook.select(index + 1);\n", + " event.shiftKey = false;\n", + " // Send a \"J\" for go to next cell\n", + " event.which = 74;\n", + " event.keyCode = 74;\n", + " manager.command_mode();\n", + " manager.handle_keydown(event);\n", " }\n", "}\n", "\n", @@ -1024,7 +1046,7 @@ { "data": { "text/html": [ - "" + "" ], "text/plain": [ "" @@ -1036,7 +1058,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "4479d7d0f95e489dbe4527908c457879", + "model_id": "b2c7e712722d4fedbe9a9fcd810f62fe", "version_major": 2, "version_minor": 0 }, @@ -1652,7 +1674,7 @@ "};\n", "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Pan axes with left mouse, zoom with right\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n", "\n", - "mpl.extensions = [\"eps\", \"jpeg\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n", + "mpl.extensions = [\"eps\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\"];\n", "\n", "mpl.default_extension = \"png\";var comm_websocket_adapter = function(comm) {\n", " // Create a \"websocket\"-like object which calls the given IPython comm\n", @@ -1822,9 +1844,12 @@ " // Check for shift+enter\n", " if (event.shiftKey && event.which == 13) {\n", " this.canvas_div.blur();\n", - " // select the cell after this one\n", - " var index = IPython.notebook.find_cell_index(this.cell_info[0]);\n", - " IPython.notebook.select(index + 1);\n", + " event.shiftKey = false;\n", + " // Send a \"J\" for go to next cell\n", + " event.which = 74;\n", + " event.keyCode = 74;\n", + " manager.command_mode();\n", + " manager.handle_keydown(event);\n", " }\n", "}\n", "\n", @@ -1873,7 +1898,7 @@ { "data": { "text/html": [ - "" + "" ], "text/plain": [ "" @@ -1894,7 +1919,7 @@ "plot(ax = ax2, **{k: background_only[v] for k,v in par_name_dict.items()})\n", "\n", "ax3.set_title(u'best fit µ = {:.3g}'.format(best_fit[pdf.config.poi_index]))\n", - "plot(ax = ax3, **{k: best_fit[v] for k,v in par_name_dict.items()})" + "plot(ax = ax3, **{k: best_fit[v] for k,v in par_name_dict.items()});" ] }, { @@ -1916,15 +1941,17 @@ "source": [ "def invert_interval(test_mus, hypo_tests, test_size=0.05):\n", " cls_obs = np.array([test[0] for test in hypo_tests]).flatten()\n", - " cls_exp = [np.array([test[1][i] for test in hypo_tests]).flatten() for i in range(5)]\n", - " crossing_test_stats = {'exp': [], 'obs': None}\n", + " cls_exp = [\n", + " np.array([test[1][i] for test in hypo_tests]).flatten() for i in range(5)\n", + " ]\n", + " crossing_test_stats = {\"exp\": [], \"obs\": None}\n", " for cls_exp_sigma in cls_exp:\n", - " crossing_test_stats['exp'].append(\n", + " crossing_test_stats[\"exp\"].append(\n", " np.interp(\n", " test_size, list(reversed(cls_exp_sigma)), list(reversed(test_mus))\n", " )\n", " )\n", - " crossing_test_stats['obs'] = np.interp(\n", + " crossing_test_stats[\"obs\"] = np.interp(\n", " test_size, list(reversed(cls_obs)), list(reversed(test_mus))\n", " )\n", " return crossing_test_stats" @@ -1933,6 +1960,26 @@ { "cell_type": "code", "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "mu_tests = np.linspace(0, 1, 16)\n", + "hypo_tests = [\n", + " pyhf.infer.hypotest(\n", + " mu,\n", + " data,\n", + " pdf,\n", + " pdf.config.suggested_init(),\n", + " pdf.config.suggested_bounds(),\n", + " return_expected_set=True,\n", + " )\n", + " for mu in mu_tests\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 13, "metadata": { "scrolled": false }, @@ -2496,7 +2543,7 @@ "};\n", "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Pan axes with left mouse, zoom with right\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n", "\n", - "mpl.extensions = [\"eps\", \"jpeg\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n", + "mpl.extensions = [\"eps\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\"];\n", "\n", "mpl.default_extension = \"png\";var comm_websocket_adapter = function(comm) {\n", " // Create a \"websocket\"-like object which calls the given IPython comm\n", @@ -2666,9 +2713,12 @@ " // Check for shift+enter\n", " if (event.shiftKey && event.which == 13) {\n", " this.canvas_div.blur();\n", - " // select the cell after this one\n", - " var index = IPython.notebook.find_cell_index(this.cell_info[0]);\n", - " IPython.notebook.select(index + 1);\n", + " event.shiftKey = false;\n", + " // Send a \"J\" for go to next cell\n", + " event.which = 74;\n", + " event.keyCode = 74;\n", + " manager.command_mode();\n", + " manager.handle_keydown(event);\n", " }\n", "}\n", "\n", @@ -2717,7 +2767,7 @@ { "data": { "text/html": [ - "" + "" ], "text/plain": [ "" @@ -2728,19 +2778,7 @@ } ], "source": [ - "mu_tests = np.linspace(0, 1, 16)\n", - "hypo_tests = [\n", - " pyhf.infer.hypotest(\n", - " mu,\n", - " data,\n", - " pdf,\n", - " pdf.config.suggested_init(),\n", - " pdf.config.suggested_bounds(),\n", - " return_expected_set=True,\n", - " )\n", - " for mu in mu_tests\n", - "]\n", - "\n", + "import pyhf.contrib.viz.brazil\n", "\n", "fig, (ax1) = plt.subplots(1, 1)\n", "fig.set_size_inches(7, 5)\n", @@ -2749,14 +2787,12 @@ "ax1.set_ylabel(u\"CLs\")\n", "ax1.set_xlabel(u\"µ\")\n", "\n", - "import pyhf.contrib.viz.brazil\n", - "\n", "pyhf.contrib.viz.brazil.plot_results(ax1, mu_tests, hypo_tests);" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -2799,7 +2835,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.6" + "version": "3.7.5" }, "widgets": { "application/vnd.jupyter.widget-state+json": { From 8fd4bbe3520226655acf0d7cc7b50484fd2b8570 Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Sun, 29 Dec 2019 23:44:03 -0600 Subject: [PATCH 6/6] Remove empty cells --- docs/examples/notebooks/hello-world.ipynb | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/docs/examples/notebooks/hello-world.ipynb b/docs/examples/notebooks/hello-world.ipynb index 8857f44504..164d68719e 100644 --- a/docs/examples/notebooks/hello-world.ipynb +++ b/docs/examples/notebooks/hello-world.ipynb @@ -136,20 +136,6 @@ "for p_value, n_sigma in enumerate(np.arange(-2,3)):\n", " print('Expected CL_s{}: {}'.format(' ' if n_sigma==0 else '({} σ)'.format(n_sigma),CLs_exp_band[p_value]))" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { @@ -168,7 +154,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.6" + "version": "3.7.5" } }, "nbformat": 4,