From e7afb7ff25550e07037052262ee27c0d964e5d39 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Wed, 28 Feb 2018 17:37:47 -0500 Subject: [PATCH] Improve py.test covered paths reporting. Use a coverage plugin instead of `coverage combine` to open the door to using a single PEX test source chroot even when testing against multiple repo source roots at once. Needed for #5426 Depends on #5420 --- 3rdparty/python/requirements.txt | 6 +- src/python/pants/backend/python/tasks/BUILD | 1 + .../pants/backend/python/tasks/coverage/BUILD | 7 + .../backend/python/tasks/coverage/__init__.py | 0 .../backend/python/tasks/coverage/plugin.py | 94 +++++++++ .../pants/backend/python/tasks/pytest_prep.py | 10 + .../pants/backend/python/tasks/pytest_run.py | 171 ++++++++++------- .../backend/python/tasks/test_pytest_run.py | 180 ++++++++++++++---- 8 files changed, 357 insertions(+), 112 deletions(-) create mode 100644 src/python/pants/backend/python/tasks/coverage/BUILD create mode 100644 src/python/pants/backend/python/tasks/coverage/__init__.py create mode 100644 src/python/pants/backend/python/tasks/coverage/plugin.py diff --git a/3rdparty/python/requirements.txt b/3rdparty/python/requirements.txt index 1becb7805c3..ca596c728f6 100644 --- a/3rdparty/python/requirements.txt +++ b/3rdparty/python/requirements.txt @@ -2,7 +2,7 @@ ansicolors==1.0.2 beautifulsoup4>=4.3.2,<4.4 cffi==1.11.1 contextlib2==0.5.5 -coverage>=4.3.4,<4.4 +coverage>=4.5,<4.6 docutils>=0.12,<0.13 fasteners==0.14.1 faulthandler==2.6 @@ -20,8 +20,8 @@ pyflakes==1.1.0 Pygments==1.4 pyopenssl==17.3.0 pystache==0.5.3 -pytest-cov>=2.4,<2.5 -pytest>=3.0.7,<4.0 +pytest-cov>=2.5,<2.6 +pytest>=3.4,<4.0 pywatchman==1.4.1 requests[security]>=2.5.0,<2.19 scandir==1.2 diff --git a/src/python/pants/backend/python/tasks/BUILD b/src/python/pants/backend/python/tasks/BUILD index 411b4ae9052..39be4a02a1b 100644 --- a/src/python/pants/backend/python/tasks/BUILD +++ b/src/python/pants/backend/python/tasks/BUILD @@ -11,6 +11,7 @@ python_library( 'src/python/pants/backend/python:interpreter_cache', 'src/python/pants/backend/python/subsystems', 'src/python/pants/backend/python/targets', + 'src/python/pants/backend/python/tasks/coverage:plugin', 'src/python/pants/base:build_environment', 'src/python/pants/base:exceptions', 'src/python/pants/base:fingerprint_strategy', diff --git a/src/python/pants/backend/python/tasks/coverage/BUILD b/src/python/pants/backend/python/tasks/coverage/BUILD new file mode 100644 index 00000000000..9410796b4dd --- /dev/null +++ b/src/python/pants/backend/python/tasks/coverage/BUILD @@ -0,0 +1,7 @@ +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +resources( + name='plugin', + sources=['plugin.py'] +) diff --git a/src/python/pants/backend/python/tasks/coverage/__init__.py b/src/python/pants/backend/python/tasks/coverage/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python/pants/backend/python/tasks/coverage/plugin.py b/src/python/pants/backend/python/tasks/coverage/plugin.py new file mode 100644 index 00000000000..3a75a2cf0d2 --- /dev/null +++ b/src/python/pants/backend/python/tasks/coverage/plugin.py @@ -0,0 +1,94 @@ +# coding=utf-8 +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +import json +import os + +from coverage import CoveragePlugin, FileTracer +from coverage.config import DEFAULT_PARTIAL, DEFAULT_PARTIAL_ALWAYS +from coverage.misc import join_regex +from coverage.parser import PythonParser +from coverage.python import PythonFileReporter + + +class MyFileTracer(FileTracer): + def __init__(self, filename): + super(MyFileTracer, self).__init__() + self._filename = filename + + def source_filename(self): + return self._filename + + +class MyFileReporter(PythonFileReporter): + """A python file reporter that knows how to map Pants PEX chroots back to repo source code.""" + + def __init__(self, morf, relpath): + super(MyFileReporter, self).__init__(morf, coverage=None) + self._relpath = relpath + + def relative_filename(self): + return self._relpath + + # TODO(John Sirois): Kill the workaround overrides below if there is a useable upstream + # resolution to: + # https://bitbucket.org/ned/coveragepy/issues/646/modifying-coverage-reporting-for-python + + @property + def parser(self): + if self._parser is None: + self._parser = PythonParser(filename=self.filename) + self._parser.parse_source() + return self._parser + + def no_branch_lines(self): + return self.parser.lines_matching(join_regex(DEFAULT_PARTIAL[:]), + join_regex(DEFAULT_PARTIAL_ALWAYS[:])) + + +class MyPlugin(CoveragePlugin): + """A plugin that knows how to map Pants PEX chroots back to repo source code when reporting.""" + + def __init__(self, buildroot, src_to_chroot): + super(MyPlugin, self).__init__() + self._buildroot = buildroot + self._src_to_chroot = src_to_chroot + + def find_executable_files(self, top): + for chroot_path in self._src_to_chroot.values(): + if top.startswith(chroot_path): + for root, _, files in os.walk(top): + for f in files: + if f.endswith('.py'): + yield os.path.join(root, f) + break + + def file_tracer(self, filename): + for chroot_path in self._src_to_chroot.values(): + if filename.startswith(chroot_path): + return MyFileTracer(filename) + + def file_reporter(self, filename): + src_file = self._map_to_src(filename) + mapped_relpath = os.path.relpath(src_file, self._buildroot) + return MyFileReporter(filename, mapped_relpath) + + def _map_to_src(self, chroot): + for src_dir, chroot_dir in self._src_to_chroot.items(): + if chroot.startswith(chroot_dir): + return src_dir + chroot[len(chroot_dir):] + raise AssertionError('Failed to map traced file {} to any source root via ' + 'source root -> chroot mappings:\n\t{}' + .format(chroot, '\n\t'.join(sorted('{} -> {}'.format(src_dir, chroot_dir) + for src_dir, chroot_dir + in self._src_to_chroot.items())))) + + +def coverage_init(reg, options): + buildroot = options['buildroot'] + src_to_chroot = json.loads(options['src_to_chroot']) + reg.add_file_tracer(MyPlugin(buildroot, src_to_chroot)) diff --git a/src/python/pants/backend/python/tasks/pytest_prep.py b/src/python/pants/backend/python/tasks/pytest_prep.py index 780fcfbb04c..890b724d0fa 100644 --- a/src/python/pants/backend/python/tasks/pytest_prep.py +++ b/src/python/pants/backend/python/tasks/pytest_prep.py @@ -7,6 +7,7 @@ import os +import pkg_resources from pex.pex_info import PexInfo from pants.backend.python.subsystems.pytest import PyTest @@ -38,6 +39,10 @@ def config_path(self): """ return os.path.join(self._pex.path(), 'pytest.ini') + @classmethod + def implementation_version(cls): + return super(PytestPrep, cls).implementation_version() + [('PytestPrep', 1)] + @classmethod def product_types(cls): return [cls.PytestBinary] @@ -52,6 +57,11 @@ def extra_requirements(self): def extra_files(self): yield self.ExtraFile.empty('pytest.ini') + enclosing_dir = os.path.dirname(__name__.replace('.', os.sep)) + plugin_path = os.path.join(enclosing_dir, 'coverage/plugin.py') + yield self.ExtraFile(path=plugin_path, + content=pkg_resources.resource_string(__name__, 'coverage/plugin.py')) + def execute(self): pex_info = PexInfo.default() pex_info.entry_point = 'pytest' diff --git a/src/python/pants/backend/python/tasks/pytest_run.py b/src/python/pants/backend/python/tasks/pytest_run.py index d4d67f83fa0..ca54573d470 100644 --- a/src/python/pants/backend/python/tasks/pytest_run.py +++ b/src/python/pants/backend/python/tasks/pytest_run.py @@ -6,6 +6,7 @@ unicode_literals, with_statement) import itertools +import json import os import shutil import time @@ -17,7 +18,6 @@ from six import StringIO from six.moves import configparser -from twitter.common.collections import OrderedSet from pants.backend.python.targets.python_tests import PythonTests from pants.backend.python.tasks.gather_sources import GatherSources @@ -91,7 +91,7 @@ class PytestRun(PartitionedTestRunnerTaskMixin, Task): @classmethod def implementation_version(cls): - return super(PytestRun, cls).implementation_version() + [('PytestRun', 2)] + return super(PytestRun, cls).implementation_version() + [('PytestRun', 3)] @classmethod def register_options(cls, register): @@ -152,7 +152,7 @@ class InvalidShardSpecification(TaskError): DEFAULT_COVERAGE_CONFIG = dedent(b""" [run] branch = True - timid = True + timid = False [report] exclude_lines = @@ -176,24 +176,32 @@ def _format_string_list(values): def _debug(self): return self.get_options().level == 'debug' - def _generate_coverage_config(self, source_mappings): + @staticmethod + def _ensure_section(cp, section): + if not cp.has_section(section): + cp.add_section(section) + + # N.B.: Extracted for tests. + @classmethod + def _add_plugin_config(cls, cp, src_to_chroot): + # We use a coverage plugin to map PEX chroot source paths back to their original repo paths for + # report output. + plugin_module = 'pants.backend.python.tasks.coverage.plugin' + cls._ensure_section(cp, 'run') + cp.set('run', 'plugins', plugin_module) + + cp.add_section(plugin_module) + cp.set(plugin_module, 'buildroot', get_buildroot()) + cp.set(plugin_module, + 'src_to_chroot', + json.dumps({os.path.join(get_buildroot(), f): os.path.join(get_buildroot(), t) + for f, t in src_to_chroot.items()})) + + def _generate_coverage_config(self, src_to_chroot): cp = configparser.SafeConfigParser() cp.readfp(StringIO(self.DEFAULT_COVERAGE_CONFIG)) - # We use the source_mappings to setup the `combine` coverage command to transform paths in - # coverage data files into canonical form. - # See the "[paths]" entry here: http://nedbatchelder.com/code/coverage/config.html for details. - cp.add_section('paths') - for canonical, alternate in source_mappings.items(): - key = canonical.replace(os.sep, '.') - - # For the benefit of macos testing, add the 'real' paths as equivalents. - paths = OrderedSet([canonical, - alternate, - os.path.realpath(canonical), - os.path.realpath(alternate)]) - - cp.set('paths', key, self._format_string_list(paths)) + self._add_plugin_config(cp, src_to_chroot) # See the debug options here: http://nedbatchelder.com/code/coverage/cmd.html#cmd-run-debug if self._debug: @@ -202,13 +210,14 @@ def _generate_coverage_config(self, source_mappings): 'config', # Logs which files are skipped or traced and why. 'trace']) + self._ensure_section(cp, 'run') cp.set('run', 'debug', debug_options) return cp @contextmanager - def _cov_setup(self, workdirs, source_mappings, coverage_sources=None): - cp = self._generate_coverage_config(source_mappings=source_mappings) + def _cov_setup(self, workdirs, coverage_morfs, src_to_chroot): + cp = self._generate_coverage_config(src_to_chroot=src_to_chroot) # Note that it's important to put the tmpfile under the workdir, because pytest # uses all arguments that look like paths to compute its rootdir, and we want # it to pick the buildroot. @@ -219,12 +228,12 @@ def _cov_setup(self, workdirs, source_mappings, coverage_sources=None): # Note that --cov-report= with no value turns off terminal reporting, which # we handle separately. args = ['--cov-report=', '--cov-config', coverage_rc] - for module in coverage_sources: - args.extend(['--cov', module]) + for morf in coverage_morfs: + args.extend(['--cov', morf]) yield args, coverage_rc @contextmanager - def _maybe_emit_coverage_data(self, workdirs, targets, pex): + def _maybe_emit_coverage_data(self, workdirs, test_targets, pex): coverage = self.get_options().coverage if coverage is None: yield [] @@ -233,18 +242,18 @@ def _maybe_emit_coverage_data(self, workdirs, targets, pex): def pex_src_root(tgt): return os.path.relpath(self._source_chroot_path((tgt,)), get_buildroot()) - source_mappings = {} - for target in targets: + src_to_chroot = {} + for target in test_targets: libs = (tgt for tgt in target.closure() if tgt.has_sources('.py') and not isinstance(tgt, PythonTests)) for lib in libs: - source_mappings[lib.target_base] = pex_src_root(lib) + src_to_chroot[lib.target_base] = pex_src_root(lib) def ensure_trailing_sep(path): return path if path.endswith(os.path.sep) else path + os.path.sep if coverage == 'auto': - def compute_coverage_sources(tgt): + def compute_coverage_pkgs(tgt): if tgt.coverage: return tgt.coverage else: @@ -255,38 +264,48 @@ def compute_coverage_sources(tgt): # but also consider supporting configuration of a global scheme whether that be parallel # dirs/packages or some arbitrary function that can be registered that takes a test target # and hands back the source packages or paths under test. - return set(os.path.dirname(s).replace(os.sep, '.') or pex_src_root(tgt) - for s in tgt.sources_relative_to_source_root()) - coverage_sources = set(itertools.chain(*[compute_coverage_sources(t) for t in targets])) + def package(test_source_path): + return os.path.dirname(test_source_path).replace(os.sep, '.') + + def packages(): + for test_source_path in tgt.sources_relative_to_source_root(): + pkg = package(test_source_path) + if pkg: + yield pkg + + return packages() + + coverage_morfs = set(itertools.chain(*[compute_coverage_pkgs(t) for t in test_targets])) else: - coverage_sources = [] - for source in coverage.split(','): - if os.path.isdir(source): + coverage_morfs = [] + for morf in coverage.split(','): + if os.path.isdir(morf): # The source is a dir, so correct its prefix for the chroot. # E.g. if source is /path/to/src/python/foo/bar or src/python/foo/bar then # rel_source is src/python/foo/bar, and ... - rel_source = os.path.relpath(source, get_buildroot()) + rel_source = os.path.relpath(morf, get_buildroot()) rel_source = ensure_trailing_sep(rel_source) + found_target_base = False - for target_base, pex_root in source_mappings.items(): + for target_base, pex_root in src_to_chroot.items(): prefix = ensure_trailing_sep(target_base) if rel_source.startswith(prefix): # ... rel_source will match on prefix=src/python/ ... suffix = rel_source[len(prefix):] # ... suffix will equal foo/bar ... - coverage_sources.append(os.path.join(pex_root, suffix)) + coverage_morfs.append(os.path.join(get_buildroot(), pex_root, suffix)) found_target_base = True # ... and we end up appending /foo/bar to the coverage_sources. break if not found_target_base: - self.context.log.warn('Coverage path {} is not in any target. Skipping.'.format(source)) + self.context.log.warn('Coverage path {} is not in any target. Skipping.'.format(morf)) else: # The source is to be interpreted as a package name. - coverage_sources.append(source) + coverage_morfs.append(morf) with self._cov_setup(workdirs, - source_mappings, - coverage_sources=coverage_sources) as (args, coverage_rc): + coverage_morfs=coverage_morfs, + src_to_chroot=src_to_chroot) as (args, coverage_rc): try: yield args finally: @@ -299,23 +318,20 @@ def coverage_run(subcommand, arguments): args=[subcommand] + arguments, env=env) - # On failures or timeouts, the .coverage file won't be written. - if not os.path.exists('.coverage'): - self.context.log.warn('No .coverage file was found! Skipping coverage reporting.') - else: - # Normalize .coverage.raw paths using combine and `paths` config in the rc file. - # This swaps the /tmp pex chroot source paths for the local original source paths - # the pex was generated from and which the user understands. - shutil.move('.coverage', '.coverage.raw') - # N.B.: This transforms the contents of .coverage.raw and moves it back into .coverage. - coverage_run('combine', ['--rcfile', coverage_rc]) + # The '.coverage' data file is output in the CWD of the test run above; so we make sure to + # look for it there. + with self._maybe_run_in_chroot(test_targets): + # On failures or timeouts, the .coverage file won't be written. + if not os.path.exists('.coverage'): + self.context.log.warn('No .coverage file was found! Skipping coverage reporting.') + else: + coverage_run('report', ['-i', '--rcfile', coverage_rc]) - coverage_run('report', ['-i', '--rcfile', coverage_rc]) + coverage_workdir = workdirs.coverage_path + coverage_run('html', ['-i', '--rcfile', coverage_rc, '-d', coverage_workdir]) - coverage_workdir = workdirs.coverage_path - coverage_run('html', ['-i', '--rcfile', coverage_rc, '-d', coverage_workdir]) - coverage_xml = os.path.join(coverage_workdir, 'coverage.xml') - coverage_run('xml', ['-i', '--rcfile', coverage_rc, '-o', coverage_xml]) + coverage_xml = os.path.join(coverage_workdir, 'coverage.xml') + coverage_run('xml', ['-i', '--rcfile', coverage_rc, '-o', coverage_xml]) def _get_shard_conftest_content(self): shard_spec = self.get_options().test_shard @@ -375,7 +391,7 @@ def _get_conftest_content(self, sources_map, rootdir_comm_path): class NodeRenamerPlugin(object): - # Map from absolute source chroot path -> buildroot relative path. + # Map from absolute source chroot path -> path of original source relative to the buildroot. _SOURCES_MAP = {sources_map!r} def __init__(self, rootdir): @@ -425,7 +441,7 @@ def _conftest(self, sources_map): rootdir_comm_path = os.path.join(conftest_dir, 'pytest_rootdir.path') def get_pytest_rootdir(): - with open(rootdir_comm_path, 'rb') as fp: + with open(rootdir_comm_path, 'r') as fp: return fp.read() conftest_content = self._get_conftest_content(sources_map, @@ -437,20 +453,29 @@ def get_pytest_rootdir(): yield conftest, get_pytest_rootdir @contextmanager - def _test_runner(self, workdirs, targets, sources_map): + def _test_runner(self, workdirs, test_targets, sources_map): pytest_binary = self.context.products.get_data(PytestPrep.PytestBinary) with self._conftest(sources_map) as (conftest, get_pytest_rootdir): - with self._maybe_emit_coverage_data(workdirs, targets, pytest_binary.pex) as coverage_args: + with self._maybe_emit_coverage_data(workdirs, + test_targets, + pytest_binary.pex) as coverage_args: yield pytest_binary, [conftest] + coverage_args, get_pytest_rootdir def _do_run_tests_with_args(self, pex, args): try: + env = dict(os.environ) + + # Ensure we don't leak source files or undeclared 3rdparty requirements into the py.test PEX + # environment. + pythonpath = env.pop('PYTHONPATH', None) + if pythonpath: + self.context.log.warn('scrubbed PYTHONPATH={} from py.test environment'.format(pythonpath)) + # The pytest runner we use accepts a --pdb argument that will launch an interactive pdb # session on any test failure. In order to support use of this pass-through flag we must # turn off stdin buffering that otherwise occurs. Setting the PYTHONUNBUFFERED env var to # any value achieves this in python2.7. We'll need a different solution when we support # running pants under CPython 3 which does not unbuffer stdin using this trick. - env = dict(os.environ) env['PYTHONUNBUFFERED'] = '1' # pytest uses py.io.terminalwriter for output. That class detects the terminal @@ -604,31 +629,31 @@ def _expose_results(self, invalid_tgts, workdirs): target_dir = os.path.join(pants_distdir, 'coverage', relpath) mergetree(workdirs.coverage_path, target_dir) - def _run_pytest(self, fail_fast, targets, workdirs): - if not targets: + def _run_pytest(self, fail_fast, test_targets, workdirs): + if not test_targets: return PytestResult.rc(0) - source_chroot_path = self._source_chroot_path(targets) + test_chroot_path = self._source_chroot_path(test_targets) - # Absolute path to chrooted source -> Path to original source relative to the buildroot. + # Absolute path to chrooted test file -> Path to original test file relative to the buildroot. sources_map = OrderedDict() - for t in targets: + for t in test_targets: for p in t.sources_relative_to_source_root(): - sources_map[os.path.join(source_chroot_path, p)] = os.path.join(t.target_base, p) + sources_map[os.path.join(test_chroot_path, p)] = os.path.join(t.target_base, p) if not sources_map: return PytestResult.rc(0) - with self._test_runner(workdirs, targets, sources_map) as (pytest_binary, - test_args, - get_pytest_rootdir): + with self._test_runner(workdirs, test_targets, sources_map) as (pytest_binary, + test_args, + get_pytest_rootdir): # Validate that the user didn't provide any passthru args that conflict # with those we must set ourselves. for arg in self.get_passthru_args(): if arg.startswith('--junitxml') or arg.startswith('--confcutdir'): raise TaskError('Cannot pass this arg through to pytest: {}'.format(arg)) - junitxml_path = workdirs.junitxml_path(*targets) + junitxml_path = workdirs.junitxml_path(*test_targets) # N.B. the `--confcutdir` here instructs pytest to stop scanning for conftest.py files at the # top of the buildroot. This prevents conftest.py files from outside (e.g. in users home dirs) @@ -653,7 +678,7 @@ def _run_pytest(self, fail_fast, targets, workdirs): if os.path.exists(junitxml_path): os.unlink(junitxml_path) - with self._maybe_run_in_chroot(targets): + with self._maybe_run_in_chroot(test_targets): result = self._do_run_tests_with_args(pytest_binary.pex, args) # There was a problem prior to test execution preventing junit xml file creation so just let @@ -663,7 +688,7 @@ def _run_pytest(self, fail_fast, targets, workdirs): pytest_rootdir = get_pytest_rootdir() failed_targets = self._get_failed_targets_from_junitxml(junitxml_path, - targets, + test_targets, pytest_rootdir) def parse_error_handler(parse_error): @@ -674,7 +699,7 @@ def parse_error_handler(parse_error): all_tests_info = self.parse_test_info(junitxml_path, parse_error_handler, ['file', 'name', 'classname']) for test_name, test_info in all_tests_info.items(): - test_target = self._get_target_from_test(test_info, targets, pytest_rootdir) + test_target = self._get_target_from_test(test_info, test_targets, pytest_rootdir) self.report_all_info_for_single_test(self.options_scope, test_target, test_name, test_info) return result.with_failed_targets(failed_targets) diff --git a/tests/python/pants_test/backend/python/tasks/test_pytest_run.py b/tests/python/pants_test/backend/python/tasks/test_pytest_run.py index 866d1e591c8..8e14453d110 100644 --- a/tests/python/pants_test/backend/python/tasks/test_pytest_run.py +++ b/tests/python/pants_test/backend/python/tasks/test_pytest_run.py @@ -9,6 +9,7 @@ from textwrap import dedent import coverage +from six.moves import configparser from pants.backend.python.targets.python_library import PythonLibrary from pants.backend.python.targets.python_tests import PythonTests @@ -18,9 +19,12 @@ from pants.backend.python.tasks.resolve_requirements import ResolveRequirements from pants.backend.python.tasks.select_interpreter import SelectInterpreter from pants.base.exceptions import ErrorWhileTesting, TaskError -from pants.util.contextutil import pushd, temporary_dir +from pants.build_graph.target import Target +from pants.source.source_root import SourceRootConfig +from pants.util.contextutil import pushd, temporary_dir, temporary_file from pants.util.dirutil import safe_mkdtemp, safe_rmtree from pants_test.backend.python.tasks.python_task_test_base import PythonTaskTestBase +from pants_test.subsystem.subsystem_util import init_subsystem from pants_test.tasks.task_test_base import ensure_cached @@ -35,12 +39,14 @@ def run_tests(self, targets, *passthru_args, **options): """Run the tests in the specified targets, with the specified PytestRun task options.""" context = self._prepare_test_run(targets, *passthru_args, **options) self._do_run_tests(context) + return context def run_failing_tests(self, targets, failed_targets, *passthru_args, **options): context = self._prepare_test_run(targets, *passthru_args, **options) with self.assertRaises(ErrorWhileTesting) as cm: self._do_run_tests(context) self.assertEqual(set(failed_targets), set(cm.exception.failed_targets)) + return context def try_run_tests(self, targets, *passthru_args, **options): try: @@ -430,28 +436,59 @@ def test_red_junit_xml_dir(self): def coverage_data_file(self): return os.path.join(self.build_root, '.coverage') - def load_coverage_data(self): + def load_coverage_data(self, context, expect_coverage=True): path = os.path.join(self.build_root, 'lib', 'core.py') - return self.load_coverage_data_for(path) + return self.load_coverage_data_for(context, path, expect_coverage=expect_coverage) - def load_coverage_data_for(self, covered_path): + def load_coverage_data_for(self, context, covered_path, expect_coverage=True): data_file = self.coverage_data_file() - self.assertTrue(os.path.isfile(data_file)) - coverage_data = coverage.coverage(data_file=data_file) - coverage_data.load() - _, all_statements, not_run_statements, _ = coverage_data.analysis(covered_path) - return all_statements, not_run_statements - - def run_coverage_auto(self, targets, failed_targets=None): + self.assertEqual(expect_coverage, os.path.isfile(data_file)) + if expect_coverage: + python_sources = context.products.get_data(GatherSources.PythonSources) + covered_relpath = os.path.relpath(covered_path, self.build_root) + owning_targets = [t for t in context.targets() + if covered_relpath in t.sources_relative_to_buildroot()] + self.assertEqual(1, len(owning_targets)) + owning_target = owning_targets[0] + + chroot = python_sources.for_target(owning_target).path() + src_root_abspath = os.path.join(self.build_root, owning_target.target_base) + covered_src_root_relpath = os.path.relpath(covered_path, src_root_abspath) + chroot_path = os.path.join(chroot, covered_src_root_relpath) + + cp = configparser.SafeConfigParser() + src_to_chroot = {os.path.join(self.build_root, tgt.target_base): + python_sources.for_target(tgt).path() + for tgt in context.targets()} + PytestRun._add_plugin_config(cp, src_to_chroot=src_to_chroot) + with temporary_file() as fp: + cp.write(fp) + fp.close() + + coverage_data = coverage.coverage(config_file=fp.name, data_file=data_file) + coverage_data.load() + + _, all_statements, not_run_statements, _ = coverage_data.analysis(chroot_path) + return all_statements, not_run_statements + + def run_coverage_auto(self, + targets, + failed_targets=None, + expect_coverage=True, + covered_path=None): self.assertFalse(os.path.isfile(self.coverage_data_file())) simple_coverage_kwargs = {'coverage': 'auto'} if failed_targets: - self.run_failing_tests(targets=targets, - failed_targets=failed_targets, - **simple_coverage_kwargs) + context = self.run_failing_tests(targets=targets, + failed_targets=failed_targets, + **simple_coverage_kwargs) else: - self.run_tests(targets=targets, **simple_coverage_kwargs) - return self.load_coverage_data() + context = self.run_tests(targets=targets, **simple_coverage_kwargs) + + if covered_path: + return self.load_coverage_data_for(context, covered_path, expect_coverage=expect_coverage) + else: + return self.load_coverage_data(context, expect_coverage=expect_coverage) @ensure_cached(PytestRun, expected_num_artifacts=1) def test_coverage_auto_option_green(self): @@ -480,23 +517,94 @@ def test_coverage_auto_option_mixed_single_target(self): self.assertEqual([1, 2, 5, 6], all_statements) self.assertEqual([], not_run_statements) - @ensure_cached(PytestRun, expected_num_artifacts=0) + @ensure_cached(PytestRun, expected_num_artifacts=1) + def test_coverage_auto_option_no_explicit_coverage(self): + init_subsystem(Target.Arguments) + init_subsystem(SourceRootConfig) + + self.create_file( + 'src/python/util/math.py', + dedent(""" + def one(): # line 1 + return 1 # line 2 + """).strip()) + util = self.make_target(spec='src/python/util', + target_type=PythonLibrary) + + self.create_file( + 'test/python/util/test_math.py', + dedent(""" + import unittest + + from util import math + + class MathTest(unittest.TestCase): + def test_one(self): + self.assertEqual(1, math.one()) + """)) + test = self.make_target(spec='test/python/util', + target_type=PythonTests, + dependencies=[util]) + covered_path = os.path.join(self.build_root, 'src/python/util/math.py') + + all_statements, not_run_statements = self.run_coverage_auto(targets=[test], + covered_path=covered_path) + self.assertEqual([1, 2], all_statements) + self.assertEqual([], not_run_statements) + + @ensure_cached(PytestRun, expected_num_artifacts=1) def test_coverage_auto_option_no_explicit_coverage_idiosyncratic_layout(self): # The all target has no coverage attribute and the code under test does not follow the - # auto-discover pattern so we should get no coverage. - all_statements, not_run_statements = self.run_coverage_auto(targets=[self.all], - failed_targets=[self.all]) - self.assertEqual([1, 2, 5, 6], all_statements) - self.assertEqual([1, 2, 5, 6], not_run_statements) + # auto-discover (parallel packages) pattern so we should get no coverage. + init_subsystem(Target.Arguments) + init_subsystem(SourceRootConfig) + + self.create_file( + 'src/python/util/math.py', + dedent(""" + def one(): # line 1 + return 1 # line 2 + """).strip()) + util = self.make_target(spec='src/python/util', + target_type=PythonLibrary) + + self.create_file( + 'test/python/util_tests/test_math.py', + dedent(""" + import unittest + + from util import math + + class MathTest(unittest.TestCase): + def test_one(self): + self.assertEqual(1, math.one()) + """)) + test = self.make_target(spec='test/python/util_tests', + target_type=PythonTests, + dependencies=[util]) + covered_path = os.path.join(self.build_root, 'src/python/util/math.py') + all_statements, not_run_statements = self.run_coverage_auto(targets=[test], + covered_path=covered_path) + self.assertEqual([1, 2], all_statements) + self.assertEqual([1, 2], not_run_statements) + + @ensure_cached(PytestRun, expected_num_artifacts=0) + def test_coverage_auto_option_no_explicit_coverage_idiosyncratic_layout_no_packages(self): + # The all target has no coverage attribute and the code under test does not follow the + # auto-discover pattern so we should get no coverage. Additionally, the all target sources + # live in the root package (they are top-level files); so they don't even have a package to use + # to guess the code under test with; as such, we should not specify and coverage sources at all, + # short-circuiting coverage. + self.run_coverage_auto(targets=[self.all], failed_targets=[self.all], expect_coverage=False) @ensure_cached(PytestRun, expected_num_artifacts=0) def test_coverage_modules_dne_option(self): self.assertFalse(os.path.isfile(self.coverage_data_file())) # Explicit modules should trump .coverage. - self.run_failing_tests(targets=[self.green, self.red], failed_targets=[self.red], - coverage='does_not_exist,nor_does_this') - all_statements, not_run_statements = self.load_coverage_data() + context = self.run_failing_tests(targets=[self.green, self.red], failed_targets=[self.red], + coverage='does_not_exist,nor_does_this') + all_statements, not_run_statements = self.load_coverage_data(context) self.assertEqual([1, 2, 5, 6], all_statements) self.assertEqual([1, 2, 5, 6], not_run_statements) @@ -504,8 +612,8 @@ def test_coverage_modules_dne_option(self): def test_coverage_modules_option(self): self.assertFalse(os.path.isfile(self.coverage_data_file())) - self.run_failing_tests(targets=[self.all], failed_targets=[self.all], coverage='core') - all_statements, not_run_statements = self.load_coverage_data() + context = self.run_failing_tests(targets=[self.all], failed_targets=[self.all], coverage='core') + all_statements, not_run_statements = self.load_coverage_data(context) self.assertEqual([1, 2, 5, 6], all_statements) self.assertEqual([], not_run_statements) @@ -513,8 +621,8 @@ def test_coverage_modules_option(self): def test_coverage_paths_option(self): self.assertFalse(os.path.isfile(self.coverage_data_file())) - self.run_failing_tests(targets=[self.all], failed_targets=[self.all], coverage='lib/') - all_statements, not_run_statements = self.load_coverage_data() + context = self.run_failing_tests(targets=[self.all], failed_targets=[self.all], coverage='lib/') + all_statements, not_run_statements = self.load_coverage_data(context) self.assertEqual([1, 2, 5, 6], all_statements) self.assertEqual([], not_run_statements) @@ -522,10 +630,10 @@ def test_coverage_paths_option(self): def test_coverage_issue_5314_primary_source_root(self): self.assertFalse(os.path.isfile(self.coverage_data_file())) - self.run_tests(targets=[self.app], coverage='app') + context = self.run_tests(targets=[self.app], coverage='app') app_path = os.path.join(self.build_root, 'app', 'app.py') - all_statements, not_run_statements = self.load_coverage_data_for(app_path) + all_statements, not_run_statements = self.load_coverage_data_for(context, app_path) self.assertEqual([1, 4, 5], all_statements) self.assertEqual([], not_run_statements) @@ -533,10 +641,10 @@ def test_coverage_issue_5314_primary_source_root(self): def test_coverage_issue_5314_secondary_source_root(self): self.assertFalse(os.path.isfile(self.coverage_data_file())) - self.run_tests(targets=[self.app], coverage='core') + context = self.run_tests(targets=[self.app], coverage='core') core_path = os.path.join(self.build_root, 'lib', 'core.py') - all_statements, not_run_statements = self.load_coverage_data_for(core_path) + all_statements, not_run_statements = self.load_coverage_data_for(context, core_path) self.assertEqual([1, 2, 5, 6], all_statements) self.assertEqual([2], not_run_statements) @@ -544,15 +652,15 @@ def test_coverage_issue_5314_secondary_source_root(self): def test_coverage_issue_5314_all_source_roots(self): self.assertFalse(os.path.isfile(self.coverage_data_file())) - self.run_tests(targets=[self.app], coverage='app,core') + context = self.run_tests(targets=[self.app], coverage='app,core') app_path = os.path.join(self.build_root, 'app', 'app.py') - all_statements, not_run_statements = self.load_coverage_data_for(app_path) + all_statements, not_run_statements = self.load_coverage_data_for(context, app_path) self.assertEqual([1, 4, 5], all_statements) self.assertEqual([], not_run_statements) core_path = os.path.join(self.build_root, 'lib', 'core.py') - all_statements, not_run_statements = self.load_coverage_data_for(core_path) + all_statements, not_run_statements = self.load_coverage_data_for(context, core_path) self.assertEqual([1, 2, 5, 6], all_statements) self.assertEqual([2], not_run_statements)