diff --git a/src/python/pants/backend/native/config/BUILD b/src/python/pants/backend/native/config/BUILD index 572ee78c077..de8355e3227 100644 --- a/src/python/pants/backend/native/config/BUILD +++ b/src/python/pants/backend/native/config/BUILD @@ -5,6 +5,7 @@ python_library( dependencies=[ '3rdparty/python:future', + 'src/python/pants/engine:isolated_process', 'src/python/pants/engine:rules', 'src/python/pants/util:objects', 'src/python/pants/util:osutil', diff --git a/src/python/pants/backend/native/config/environment.py b/src/python/pants/backend/native/config/environment.py index a72c684cd68..3c6076fe8a4 100644 --- a/src/python/pants/backend/native/config/environment.py +++ b/src/python/pants/backend/native/config/environment.py @@ -8,6 +8,7 @@ from abc import abstractproperty from builtins import object +from pants.engine.isolated_process import ExecuteProcessRequest from pants.engine.rules import SingletonRule from pants.util.objects import datatype from pants.util.osutil import all_normalized_os_names, get_normalized_os_name @@ -44,19 +45,21 @@ def resolve_platform_specific(self, platform_specific_funs): class Executable(object): - @abstractproperty + @property def path_entries(self): """A list of directory paths containing this executable, to be used in a subprocess's PATH. This may be multiple directories, e.g. if the main executable program invokes any subprocesses. """ + return [] - @abstractproperty - def library_dirs(self): + @property + def runtime_library_dirs(self): """Directories containing shared libraries that must be on the runtime library search path. Note: this is for libraries needed for the current Executable to run -- see LinkerMixin below for libraries that are needed at link time.""" + return [] @abstractproperty def exe_filename(self): @@ -76,14 +79,21 @@ def as_invocation_environment_dict(self): }) return { 'PATH': create_path_env_var(self.path_entries), - lib_env_var: create_path_env_var(self.library_dirs), + lib_env_var: create_path_env_var(self.runtime_library_dirs), } + def as_execute_process_request(self, more_args=None): + argv = [self.exe_filename] + self.extra_args + (more_args or []) + return ExecuteProcessRequest.create_with_empty_snapshot( + argv=tuple(argv), + description=repr(self), + env=self.as_invocation_environment_dict) + class Assembler(datatype([ 'path_entries', 'exe_filename', - 'library_dirs', + 'runtime_library_dirs', ]), Executable): pass @@ -109,10 +119,26 @@ def as_invocation_environment_dict(self): class Linker(datatype([ 'path_entries', 'exe_filename', - 'library_dirs', + 'runtime_library_dirs', 'linking_library_dirs', 'extra_args', -]), LinkerMixin): pass +]), LinkerMixin): + + @property + def with_tupled_collections(self): + """???""" + # FIXME: convert these to using `copy()` when #6269 is merged. + return Linker( + path_entries=tuple(self.path_entries), + exe_filename=self.exe_filename, + runtime_library_dirs=tuple(self.runtime_library_dirs), + linking_library_dirs=tuple(self.linking_library_dirs), + extra_args=tuple(self.extra_args)) + # return self.copy( + # path_entries=tuple(self.path_entries), + # runtime_library_dirs=tuple(self.runtime_library_dirs), + # linking_library_dirs=tuple(self.linking_library_dirs), + # extra_args=tuple(self.extra_args)) class CompilerMixin(Executable): @@ -134,7 +160,7 @@ def as_invocation_environment_dict(self): class CCompiler(datatype([ 'path_entries', 'exe_filename', - 'library_dirs', + 'runtime_library_dirs', 'include_dirs', 'extra_args', ]), CompilerMixin): @@ -143,15 +169,35 @@ class CCompiler(datatype([ def as_invocation_environment_dict(self): ret = super(CCompiler, self).as_invocation_environment_dict.copy() - ret['CC'] = self.exe_filename + ret.update({ + 'CC': self.exe_filename, + # This avoids decoding errors parsing unicode smart quotes that gcc outputs for fun. + 'LC_ALL': 'C', + }) return ret + @property + def with_tupled_collections(self): + """???""" + # FIXME: convert these to using `copy()` when #6269 is merged. + return CCompiler( + path_entries=tuple(self.path_entries), + exe_filename=self.exe_filename, + runtime_library_dirs=tuple(self.runtime_library_dirs), + include_dirs=tuple(self.include_dirs), + extra_args=tuple(self.extra_args)) + # return self.copy( + # path_entries=tuple(self.path_entries), + # runtime_library_dirs=tuple(self.runtime_library_dirs), + # include_dirs=tuple(self.include_dirs), + # extra_args=tuple(self.extra_args)) + class CppCompiler(datatype([ 'path_entries', 'exe_filename', - 'library_dirs', + 'runtime_library_dirs', 'include_dirs', 'extra_args', ]), CompilerMixin): @@ -160,10 +206,30 @@ class CppCompiler(datatype([ def as_invocation_environment_dict(self): ret = super(CppCompiler, self).as_invocation_environment_dict.copy() - ret['CXX'] = self.exe_filename + ret.update({ + 'CXX': self.exe_filename, + # This avoids decoding errors parsing unicode smart quotes that gcc outputs for fun. + 'LC_ALL': 'C', + }) return ret + @property + def with_tupled_collections(self): + """???""" + # FIXME: convert these to using `copy()` when #6269 is merged. + return CppCompiler( + path_entries=tuple(self.path_entries), + exe_filename=self.exe_filename, + runtime_library_dirs=tuple(self.runtime_library_dirs), + include_dirs=tuple(self.include_dirs), + extra_args=tuple(self.extra_args)) + # return self.copy( + # path_entries=tuple(self.path_entries), + # runtime_library_dirs=tuple(self.runtime_library_dirs), + # include_dirs=tuple(self.include_dirs), + # extra_args=tuple(self.extra_args)) + # NB: These wrapper classes for LLVM and GCC toolchains are performing the work of variants. A # CToolchain cannot be requested directly, but native_toolchain.py provides an LLVMCToolchain, diff --git a/src/python/pants/backend/native/subsystems/binaries/binutils.py b/src/python/pants/backend/native/subsystems/binaries/binutils.py index 63c554261b7..8421d7fecb2 100644 --- a/src/python/pants/backend/native/subsystems/binaries/binutils.py +++ b/src/python/pants/backend/native/subsystems/binaries/binutils.py @@ -24,13 +24,13 @@ def assembler(self): return Assembler( path_entries=self.path_entries(), exe_filename='as', - library_dirs=[]) + runtime_library_dirs=[]) def linker(self): return Linker( path_entries=self.path_entries(), exe_filename='ld', - library_dirs=[], + runtime_library_dirs=[], linking_library_dirs=[], extra_args=[]) diff --git a/src/python/pants/backend/native/subsystems/binaries/gcc.py b/src/python/pants/backend/native/subsystems/binaries/gcc.py index 0d2b253f82a..36785962b75 100644 --- a/src/python/pants/backend/native/subsystems/binaries/gcc.py +++ b/src/python/pants/backend/native/subsystems/binaries/gcc.py @@ -65,7 +65,7 @@ def c_compiler(self, platform): return CCompiler( path_entries=self.path_entries, exe_filename='gcc', - library_dirs=self._common_lib_dirs(platform), + runtime_library_dirs=self._common_lib_dirs(platform), include_dirs=self._common_include_dirs, extra_args=[]) @@ -89,7 +89,7 @@ def cpp_compiler(self, platform): return CppCompiler( path_entries=self.path_entries, exe_filename='g++', - library_dirs=self._common_lib_dirs(platform), + runtime_library_dirs=self._common_lib_dirs(platform), include_dirs=(self._common_include_dirs + self._cpp_include_dirs), extra_args=[]) diff --git a/src/python/pants/backend/native/subsystems/binaries/llvm.py b/src/python/pants/backend/native/subsystems/binaries/llvm.py index fe6476d97aa..5edf52c43bf 100644 --- a/src/python/pants/backend/native/subsystems/binaries/llvm.py +++ b/src/python/pants/backend/native/subsystems/binaries/llvm.py @@ -90,7 +90,7 @@ def linker(self, platform): path_entries=self.path_entries, exe_filename=platform.resolve_platform_specific( self._PLATFORM_SPECIFIC_LINKER_NAME), - library_dirs=[], + runtime_library_dirs=[], linking_library_dirs=[], extra_args=[]) @@ -106,7 +106,7 @@ def c_compiler(self): return CCompiler( path_entries=self.path_entries, exe_filename='clang', - library_dirs=self._common_lib_dirs, + runtime_library_dirs=self._common_lib_dirs, include_dirs=self._common_include_dirs, extra_args=[]) @@ -118,7 +118,7 @@ def cpp_compiler(self): return CppCompiler( path_entries=self.path_entries, exe_filename='clang++', - library_dirs=self._common_lib_dirs, + runtime_library_dirs=self._common_lib_dirs, include_dirs=(self._cpp_include_dirs + self._common_include_dirs), extra_args=[]) diff --git a/src/python/pants/backend/native/subsystems/libc_dev.py b/src/python/pants/backend/native/subsystems/libc_dev.py deleted file mode 100644 index 80599b80a3f..00000000000 --- a/src/python/pants/backend/native/subsystems/libc_dev.py +++ /dev/null @@ -1,109 +0,0 @@ -# coding=utf-8 -# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import absolute_import, division, print_function, unicode_literals - -import os - -from pants.backend.native.config.environment import HostLibcDev -from pants.backend.native.subsystems.utils.parse_search_dirs import ParseSearchDirs -from pants.base.hash_utils import hash_file -from pants.option.custom_types import dir_option -from pants.subsystem.subsystem import Subsystem -from pants.util.memo import memoized_property - - -class LibcDev(Subsystem): - """Subsystem to detect and provide the host's installed version of a libc "dev" package. - - A libc "dev" package is provided on most Linux systems by default, but may not be located at any - standardized path. We define a libc dev package as one which provides crti.o, an object file which - is part of any libc implementation and is required to create executables (more information - available at https://wiki.osdev.org/Creating_a_C_Library). - - NB: This is currently unused except in CI, because we have no plans to support creating native - executables from C or C++ sources yet (PRs welcome!). It is used to provide an "end-to-end" test - of the compilation and linking toolchain in CI by creating and invoking a "hello world" - executable. - """ - - options_scope = 'libc' - - class HostLibcDevResolutionError(Exception): pass - - @classmethod - def subsystem_dependencies(cls): - return super(LibcDev, cls).subsystem_dependencies() + (ParseSearchDirs.scoped(cls),) - - @memoized_property - def _parse_search_dirs(self): - return ParseSearchDirs.scoped_instance(self) - - @classmethod - def register_options(cls, register): - super(LibcDev, cls).register_options(register) - - register('--enable-libc-search', type=bool, default=False, fingerprint=True, advanced=True, - help="Whether to search for the host's libc installation. Set to False if the host " - "does not have a libc install with crti.o -- this file is necessary to create " - "executables on Linux hosts.") - register('--libc-dir', type=dir_option, default=None, fingerprint=True, advanced=True, - help='A directory containing a host-specific crti.o from libc.') - register('--host-compiler', type=str, default='gcc', fingerprint=True, advanced=True, - help='The host compiler to invoke with -print-search-dirs to find the host libc.') - - # NB: crti.o is required to create executables on Linux. Our provided gcc and clang can find it if - # the containing directory is within the LIBRARY_PATH environment variable when we invoke the - # compiler. - _LIBC_INIT_OBJECT_FILE = 'crti.o' - - def _get_host_libc_from_host_compiler(self): - """Locate the host's libc-dev installation using a specified host compiler's search dirs.""" - compiler_exe = self.get_options().host_compiler - - # Implicitly, we are passing in the environment of the executing pants process to - # `get_compiler_library_dirs()`. - # These directories are checked to exist! - library_dirs = self._parse_search_dirs.get_compiler_library_dirs(compiler_exe) - - libc_crti_object_file = None - for libc_dir_candidate in library_dirs: - maybe_libc_crti = os.path.join(libc_dir_candidate, self._LIBC_INIT_OBJECT_FILE) - if os.path.isfile(maybe_libc_crti): - libc_crti_object_file = maybe_libc_crti - break - - if not libc_crti_object_file: - raise self.HostLibcDevResolutionError( - "Could not locate {fname} in library search dirs {dirs} from compiler: {compiler!r}. " - "You may need to install a libc dev package for the current system. " - "For many operating systems, this package is named 'libc-dev' or 'libc6-dev'." - .format(fname=self._LIBC_INIT_OBJECT_FILE, dirs=library_dirs, compiler=compiler_exe)) - - return HostLibcDev(crti_object=libc_crti_object_file, - fingerprint=hash_file(libc_crti_object_file)) - - @memoized_property - def _host_libc(self): - """Use the --libc-dir option if provided, otherwise invoke a host compiler to find libc dev.""" - libc_dir_option = self.get_options().libc_dir - if libc_dir_option: - maybe_libc_crti = os.path.join(libc_dir_option, self._LIBC_INIT_OBJECT_FILE) - if os.path.isfile(maybe_libc_crti): - return HostLibcDev(crti_object=maybe_libc_crti, - fingerprint=hash_file(maybe_libc_crti)) - raise self.HostLibcDevResolutionError( - "Could not locate {} in directory {} provided by the --libc-dir option." - .format(self._LIBC_INIT_OBJECT_FILE, libc_dir_option)) - - return self._get_host_libc_from_host_compiler() - - def get_libc_dirs(self, platform): - if not self.get_options().enable_libc_search: - return [] - - return platform.resolve_platform_specific({ - 'darwin': lambda: [], - 'linux': lambda: [self._host_libc.get_lib_dir()], - }) diff --git a/src/python/pants/backend/native/subsystems/native_toolchain.py b/src/python/pants/backend/native/subsystems/native_toolchain.py index ddf388364d5..e074e78f23b 100644 --- a/src/python/pants/backend/native/subsystems/native_toolchain.py +++ b/src/python/pants/backend/native/subsystems/native_toolchain.py @@ -4,20 +4,30 @@ from __future__ import absolute_import, division, print_function, unicode_literals -from pants.backend.native.config.environment import (Assembler, CCompiler, CppCompiler, - CppToolchain, CToolchain, GCCCppToolchain, - GCCCToolchain, Linker, LLVMCppToolchain, - LLVMCToolchain, Platform) +import logging +import os +import re + +from twitter.common.collections import OrderedSet + +from pants.backend.native.config.environment import (Assembler, CCompiler, CompilerMixin, + CppCompiler, CppToolchain, CToolchain, + GCCCppToolchain, GCCCToolchain, Linker, + LLVMCppToolchain, LLVMCToolchain, Platform) from pants.backend.native.subsystems.binaries.binutils import Binutils from pants.backend.native.subsystems.binaries.gcc import GCC from pants.backend.native.subsystems.binaries.llvm import LLVM -from pants.backend.native.subsystems.libc_dev import LibcDev from pants.backend.native.subsystems.xcode_cli_tools import XCodeCLITools +from pants.engine.isolated_process import ExecuteProcessRequest, ExecuteProcessResult from pants.engine.rules import RootRule, rule from pants.engine.selectors import Get, Select from pants.subsystem.subsystem import Subsystem +from pants.util.dirutil import is_readable_dir from pants.util.memo import memoized_property -from pants.util.objects import datatype +from pants.util.objects import SubclassesOf, datatype + + +logger = logging.getLogger(__name__) class NativeToolchain(Subsystem): @@ -39,7 +49,6 @@ def subsystem_dependencies(cls): return super(NativeToolchain, cls).subsystem_dependencies() + ( Binutils.scoped(cls), GCC.scoped(cls), - LibcDev.scoped(cls), LLVM.scoped(cls), XCodeCLITools.scoped(cls), ) @@ -60,14 +69,158 @@ def _llvm(self): def _xcode_cli_tools(self): return XCodeCLITools.scoped_instance(self) - @memoized_property - def _libc_dev(self): - return LibcDev.scoped_instance(self) +class CompilerSystemDirSearchError(Exception): + """Thrown for errors in finding system includes and lib dirs for a compiler.""" + + +class DirCollectionRequest(datatype([('dir_paths', tuple)])): pass + + +class ExistingDirCollection(datatype([('dirs', tuple)])): pass + + +# TODO: use snapshots for this!!! +@rule(ExistingDirCollection, [Select(DirCollectionRequest)]) +def filter_existing_dirs(dir_collection_request): + real_dirs = OrderedSet() + for maybe_existing_dir in dir_collection_request.dir_paths: + # Could use a `seen_dir_paths` set if we want to avoid pinging the fs for duplicate entries. + if is_readable_dir(maybe_existing_dir): + real_dirs.add(os.path.realpath(maybe_existing_dir)) + else: + logger.debug("found non-existent or non-accessible directory '{}' from request {}" + .format(maybe_existing_dir, dir_collection_request)) + + + return ExistingDirCollection(dirs=tuple(real_dirs)) + + +class CompilerSearchRequest(datatype([('compiler', SubclassesOf(CompilerMixin))])): pass + + +class LibDirsFromCompiler(ExistingDirCollection): pass + + +_search_dirs_libraries_regex = re.compile('^libraries: =(.*)$', flags=re.MULTILINE) + + +@rule(LibDirsFromCompiler, [Select(CompilerSearchRequest)]) +def parse_known_lib_dirs(compiler_search_request): + print_search_dirs_exe = compiler_search_request.compiler.copy(extra_args=['-print-search-dirs']) + exe_response = yield Get( + ExecuteProcessResult, + ExecuteProcessRequest, + print_search_dirs_exe.as_execute_process_request()) + + compiler_output = exe_response.stdout + exe_response.stderr + libs_line = _search_dirs_libraries_regex.search(compiler_output) + if not libs_line: + raise CompilerSystemDirSearchError( + "Could not parse libraries for compiler search request {!r}. Output:\n{}" + .format(compiler_search_request, compiler_output)) + + dir_collection_request = DirCollectionRequest(dir_paths=tuple(libs_line.group(1).split(':'))) + existing_dir_collection = yield Get( + ExistingDirCollection, + DirCollectionRequest, + dir_collection_request) + + yield LibDirsFromCompiler(dirs=existing_dir_collection.dirs) + + +class IncludeDirsFromCompiler(ExistingDirCollection): pass + + +_include_dir_paths_start_line = '#include <...> search starts here:' +_include_dir_paths_end_line = 'End of search list.' + + +@rule(IncludeDirsFromCompiler, [Select(CompilerSearchRequest)]) +def parse_known_include_dirs(compiler_search_request): + print_include_search_exe = compiler_search_request.compiler.copy(extra_args=['-E', '-Wp,-v', '-']) + exe_response = yield Get( + ExecuteProcessResult, + ExecuteProcessRequest, + print_include_search_exe.as_execute_process_request()) + + compiler_output = exe_response.stdout + exe_response.stderr + parsed_include_paths = None + for output_line in compiler_output.split('\n'): + if output_line == _include_dir_paths_start_line: + parsed_include_paths = [] + continue + elif output_line == _include_dir_paths_end_line: + break + + if parsed_include_paths is not None: + # Each line starts with a single initial space. + parsed_include_paths.append(output_line[1:]) + + dir_collection_request = DirCollectionRequest(dir_paths=tuple(parsed_include_paths)) + existing_dir_collection = yield Get( + ExistingDirCollection, + DirCollectionRequest, + dir_collection_request) + + yield IncludeDirsFromCompiler(dirs=existing_dir_collection.dirs) + + +class CompilerSearchOutput(datatype([ + ('lib_dirs', LibDirsFromCompiler), + ('include_dirs', IncludeDirsFromCompiler), +])): pass + + +@rule(CompilerSearchOutput, [Select(CompilerSearchRequest)]) +def get_compiler_resources(compiler_search_request): + lib_dirs = yield Get(LibDirsFromCompiler, CompilerSearchRequest, compiler_search_request) + include_dirs = yield Get(IncludeDirsFromCompiler, CompilerSearchRequest, compiler_search_request) + yield CompilerSearchOutput(lib_dirs=lib_dirs, include_dirs=include_dirs) + + +class CToolchainRequest(datatype([ + ('c_compiler', CCompiler), + ('c_linker', Linker) +])): pass + + +@rule(CToolchain, [Select(CToolchainRequest)]) +def resolve_c_toolchain(c_toolchain_request): + c_compiler = c_toolchain_request.c_compiler + c_linker = c_toolchain_request.c_linker + compiler_search_output = yield Get( + CompilerSearchOutput, + CompilerSearchRequest, + CompilerSearchRequest(compiler=c_compiler)) + + resolved_c_toolchain = CToolchain( + c_compiler=c_compiler.copy(include_dirs=compiler_search_output.include_dirs.dirs), + c_linker=c_linker.copy(linking_library_dirs=compiler_search_output.lib_dirs.dirs)) + + yield resolved_c_toolchain + + +class CppToolchainRequest(datatype([ + ('cpp_compiler', CppCompiler), + ('cpp_linker', Linker) +])): pass + + +@rule(CppToolchain, [Select(CppToolchainRequest)]) +def resolve_cpp_toolchain(cpp_toolchain_request): + cpp_compiler = cpp_toolchain_request.cpp_compiler + cpp_linker = cpp_toolchain_request.cpp_linker + compiler_search_output = yield Get( + CompilerSearchOutput, + CompilerSearchRequest, + CompilerSearchRequest(compiler=cpp_compiler)) + + resolved_cpp_toolchain = CppToolchain( + cpp_compiler=cpp_compiler.copy(include_dirs=compiler_search_output.include_dirs.dirs), + cpp_linker=cpp_linker.copy(linking_library_dirs=compiler_search_output.lib_dirs.dirs)) -@rule(LibcDev, [Select(NativeToolchain)]) -def select_libc_dev(native_toolchain): - yield native_toolchain._libc_dev + yield resolved_cpp_toolchain @rule(Assembler, [Select(Platform), Select(NativeToolchain)]) @@ -121,35 +274,39 @@ def select_llvm_c_toolchain(platform, native_toolchain): # These arguments are shared across platforms. llvm_c_compiler_args = [ '-x', 'c', '-std=c11', - '-nobuiltininc', + '-nostdinc', ] if platform.normalized_os_name == 'darwin': xcode_clang = yield Get(CCompiler, XCodeCLITools, native_toolchain._xcode_cli_tools) working_c_compiler = provided_clang.copy( path_entries=(provided_clang.path_entries + xcode_clang.path_entries), - library_dirs=(provided_clang.library_dirs + xcode_clang.library_dirs), + runtime_library_dirs=(provided_clang.runtime_library_dirs + xcode_clang.runtime_library_dirs), include_dirs=(provided_clang.include_dirs + xcode_clang.include_dirs), - extra_args=(provided_clang.extra_args + llvm_c_compiler_args + xcode_clang.extra_args)) + extra_args=(llvm_c_compiler_args + xcode_clang.extra_args + ['-nobuiltininc'])) else: gcc_install = yield Get(GCCInstallLocationForLLVM, GCC, native_toolchain._gcc) provided_gcc = yield Get(CCompiler, GCC, native_toolchain._gcc) working_c_compiler = provided_clang.copy( # We need g++'s version of the GLIBCXX library to be able to run, unfortunately. - library_dirs=(provided_gcc.library_dirs + provided_clang.library_dirs), + runtime_library_dirs=(provided_gcc.runtime_library_dirs + + provided_clang.runtime_library_dirs), include_dirs=provided_gcc.include_dirs, extra_args=(llvm_c_compiler_args + provided_clang.extra_args + gcc_install.as_clang_argv)) base_linker_wrapper = yield Get(BaseLinker, NativeToolchain, native_toolchain) base_linker = base_linker_wrapper.linker - libc_dev = yield Get(LibcDev, NativeToolchain, native_toolchain) working_linker = base_linker.copy( path_entries=(base_linker.path_entries + working_c_compiler.path_entries), exe_filename=working_c_compiler.exe_filename, - library_dirs=(base_linker.library_dirs + working_c_compiler.library_dirs), - linking_library_dirs=(base_linker.linking_library_dirs + libc_dev.get_libc_dirs(platform))) + runtime_library_dirs=(base_linker.runtime_library_dirs + + working_c_compiler.runtime_library_dirs)) - yield LLVMCToolchain(CToolchain(working_c_compiler, working_linker)) + c_toolchain_request = CToolchainRequest( + c_compiler=working_c_compiler.with_tupled_collections, + c_linker=working_linker.with_tupled_collections) + resolved_toolchain = yield Get(CToolchain, CToolchainRequest, c_toolchain_request) + yield LLVMCToolchain(resolved_toolchain) @rule(LLVMCppToolchain, [Select(Platform), Select(NativeToolchain)]) @@ -162,7 +319,7 @@ def select_llvm_cpp_toolchain(platform, native_toolchain): # This mean we don't use any of the headers from our LLVM distribution's C++ stdlib # implementation, or any from the host system. Instead, we use include dirs from the # XCodeCLITools or GCC. - '-nobuiltininc', + '-nostdinc', '-nostdinc++', ] @@ -170,11 +327,12 @@ def select_llvm_cpp_toolchain(platform, native_toolchain): xcode_clang = yield Get(CppCompiler, XCodeCLITools, native_toolchain._xcode_cli_tools) working_cpp_compiler = provided_clangpp.copy( path_entries=(provided_clangpp.path_entries + xcode_clang.path_entries), - library_dirs=(provided_clangpp.library_dirs + xcode_clang.library_dirs), + runtime_library_dirs=(provided_clangpp.runtime_library_dirs + + xcode_clang.runtime_library_dirs), include_dirs=(provided_clangpp.include_dirs + xcode_clang.include_dirs), # On OSX, this uses the libc++ (LLVM) C++ standard library implementation. This is # feature-complete for OSX, but not for Linux (see https://libcxx.llvm.org/ for more info). - extra_args=(llvm_cpp_compiler_args + provided_clangpp.extra_args + xcode_clang.extra_args)) + extra_args=(llvm_cpp_compiler_args + xcode_clang.extra_args + ['-nobuiltininc'])) linking_library_dirs = [] linker_extra_args = [] else: @@ -182,27 +340,29 @@ def select_llvm_cpp_toolchain(platform, native_toolchain): provided_gpp = yield Get(CppCompiler, GCC, native_toolchain._gcc) working_cpp_compiler = provided_clangpp.copy( # We need g++'s version of the GLIBCXX library to be able to run, unfortunately. - library_dirs=(provided_gpp.library_dirs + provided_clangpp.library_dirs), + runtime_library_dirs=(provided_gpp.runtime_library_dirs + provided_clangpp.runtime_library_dirs), # NB: we use g++'s headers on Linux, and therefore their C++ standard library. include_dirs=provided_gpp.include_dirs, - extra_args=(llvm_cpp_compiler_args + provided_clangpp.extra_args + gcc_install.as_clang_argv)) - linking_library_dirs = provided_gpp.library_dirs + provided_clangpp.library_dirs + extra_args=(llvm_cpp_compiler_args + gcc_install.as_clang_argv)) + linking_library_dirs = provided_gpp.runtime_library_dirs + provided_clangpp.runtime_library_dirs # Ensure we use libstdc++, provided by g++, during the linking stage. linker_extra_args=['-stdlib=libstdc++'] - libc_dev = yield Get(LibcDev, NativeToolchain, native_toolchain) base_linker_wrapper = yield Get(BaseLinker, NativeToolchain, native_toolchain) base_linker = base_linker_wrapper.linker working_linker = base_linker.copy( path_entries=(base_linker.path_entries + working_cpp_compiler.path_entries), exe_filename=working_cpp_compiler.exe_filename, - library_dirs=(base_linker.library_dirs + working_cpp_compiler.library_dirs), + runtime_library_dirs=(base_linker.runtime_library_dirs + working_cpp_compiler.runtime_library_dirs), linking_library_dirs=(base_linker.linking_library_dirs + - linking_library_dirs + - libc_dev.get_libc_dirs(platform)), + linking_library_dirs), extra_args=(base_linker.extra_args + linker_extra_args)) - yield LLVMCppToolchain(CppToolchain(working_cpp_compiler, working_linker)) + cpp_toolchain_request = CppToolchainRequest( + cpp_compiler=working_cpp_compiler.with_tupled_collections, + cpp_linker=working_linker.with_tupled_collections) + resolved_toolchain = yield Get(CppToolchain, CppToolchainRequest, cpp_toolchain_request) + yield LLVMCppToolchain(resolved_toolchain) @rule(GCCCToolchain, [Select(Platform), Select(NativeToolchain)]) @@ -226,18 +386,22 @@ def select_gcc_c_toolchain(platform, native_toolchain): working_c_compiler = provided_gcc.copy( path_entries=(provided_gcc.path_entries + assembler.path_entries), include_dirs=new_include_dirs, - extra_args=['-x', 'c', '-std=c11']) + extra_args=['-x', 'c', '-std=c11', '-nostdinc']) base_linker_wrapper = yield Get(BaseLinker, NativeToolchain, native_toolchain) base_linker = base_linker_wrapper.linker - libc_dev = yield Get(LibcDev, NativeToolchain, native_toolchain) working_linker = base_linker.copy( path_entries=(working_c_compiler.path_entries + base_linker.path_entries), exe_filename=working_c_compiler.exe_filename, - library_dirs=(base_linker.library_dirs + working_c_compiler.library_dirs), - linking_library_dirs=(base_linker.linking_library_dirs + libc_dev.get_libc_dirs(platform))) + runtime_library_dirs=(base_linker.runtime_library_dirs + + working_c_compiler.runtime_library_dirs), + linking_library_dirs=(base_linker.linking_library_dirs)) - yield GCCCToolchain(CToolchain(working_c_compiler, working_linker)) + c_toolchain_request = CToolchainRequest( + c_compiler=working_c_compiler.with_tupled_collections, + c_linker=working_linker.with_tupled_collections) + resolved_toolchain = yield Get(CToolchain, CToolchainRequest, c_toolchain_request) + yield GCCCToolchain(resolved_toolchain) @rule(GCCCppToolchain, [Select(Platform), Select(NativeToolchain)]) @@ -263,24 +427,37 @@ def select_gcc_cpp_toolchain(platform, native_toolchain): include_dirs=new_include_dirs, extra_args=([ '-x', 'c++', '-std=c++11', + '-nostdinc', '-nostdinc++', ])) base_linker_wrapper = yield Get(BaseLinker, NativeToolchain, native_toolchain) base_linker = base_linker_wrapper.linker - libc_dev = yield Get(LibcDev, NativeToolchain, native_toolchain) working_linker = base_linker.copy( path_entries=(working_cpp_compiler.path_entries + base_linker.path_entries), exe_filename=working_cpp_compiler.exe_filename, - library_dirs=(base_linker.library_dirs + working_cpp_compiler.library_dirs), - linking_library_dirs=(base_linker.linking_library_dirs + libc_dev.get_libc_dirs(platform))) + runtime_library_dirs=(base_linker.runtime_library_dirs + + working_cpp_compiler.runtime_library_dirs)) - yield GCCCppToolchain(CppToolchain(working_cpp_compiler, working_linker)) + cpp_toolchain_request = CppToolchainRequest( + cpp_compiler=working_cpp_compiler.with_tupled_collections, + cpp_linker=working_linker.with_tupled_collections) + resolved_toolchain = yield Get(CppToolchain, CppToolchainRequest, cpp_toolchain_request) + yield GCCCppToolchain(resolved_toolchain) def create_native_toolchain_rules(): return [ - select_libc_dev, + filter_existing_dirs, + RootRule(DirCollectionRequest), + parse_known_lib_dirs, + parse_known_include_dirs, + get_compiler_resources, + RootRule(CompilerSearchRequest), + resolve_c_toolchain, + RootRule(CToolchainRequest), + resolve_cpp_toolchain, + RootRule(CppToolchainRequest), select_assembler, select_base_linker, select_gcc_install_location, diff --git a/src/python/pants/backend/native/subsystems/xcode_cli_tools.py b/src/python/pants/backend/native/subsystems/xcode_cli_tools.py index 08bad5515cb..d441a06b8d4 100644 --- a/src/python/pants/backend/native/subsystems/xcode_cli_tools.py +++ b/src/python/pants/backend/native/subsystems/xcode_cli_tools.py @@ -140,14 +140,14 @@ def assembler(self): return Assembler( path_entries=self.path_entries(), exe_filename='as', - library_dirs=[]) + runtime_library_dirs=[]) @memoized_method def linker(self): return Linker( path_entries=self.path_entries(), exe_filename='ld', - library_dirs=[], + runtime_library_dirs=[], linking_library_dirs=[], extra_args=[MIN_OSX_VERSION_ARG]) @@ -156,7 +156,7 @@ def c_compiler(self): return CCompiler( path_entries=self.path_entries(), exe_filename='clang', - library_dirs=self.lib_dirs(), + runtime_library_dirs=self.lib_dirs(), include_dirs=self.include_dirs(), extra_args=[MIN_OSX_VERSION_ARG]) @@ -165,7 +165,7 @@ def cpp_compiler(self): return CppCompiler( path_entries=self.path_entries(), exe_filename='clang++', - library_dirs=self.lib_dirs(), + runtime_library_dirs=self.lib_dirs(), include_dirs=self.include_dirs(), extra_args=[MIN_OSX_VERSION_ARG]) diff --git a/src/python/pants/backend/native/tasks/link_shared_libraries.py b/src/python/pants/backend/native/tasks/link_shared_libraries.py index 965631ff7b0..eb69f3bec7b 100644 --- a/src/python/pants/backend/native/tasks/link_shared_libraries.py +++ b/src/python/pants/backend/native/tasks/link_shared_libraries.py @@ -198,7 +198,7 @@ def _execute_link_request(self, link_request): # We are executing in the results_dir, so get absolute paths for everything. cmd = ([linker.exe_filename] + self.platform.resolve_platform_specific(self._SHARED_CMDLINE_ARGS) + - linker.extra_args + + list(linker.extra_args) + ['-o', os.path.abspath(resulting_shared_lib_path)] + [os.path.abspath(obj) for obj in object_files]) self.context.log.debug("linker command: {}".format(cmd)) diff --git a/src/python/pants/backend/native/tasks/native_compile.py b/src/python/pants/backend/native/tasks/native_compile.py index 2efe955b0dd..614611efcb6 100644 --- a/src/python/pants/backend/native/tasks/native_compile.py +++ b/src/python/pants/backend/native/tasks/native_compile.py @@ -225,7 +225,7 @@ def _make_compile_argv(self, compile_request): buildroot = get_buildroot() argv = ( [compiler.exe_filename] + - compiler.extra_args + + list(compiler.extra_args) + err_flags + ['-c', '-fPIC'] + [ diff --git a/src/python/pants/backend/python/subsystems/python_native_code.py b/src/python/pants/backend/python/subsystems/python_native_code.py index cfa772679ed..6bb8eccbe15 100644 --- a/src/python/pants/backend/python/subsystems/python_native_code.py +++ b/src/python/pants/backend/python/subsystems/python_native_code.py @@ -4,6 +4,7 @@ from __future__ import absolute_import, division, print_function, unicode_literals +import logging import os from builtins import str from collections import defaultdict @@ -25,6 +26,9 @@ from pants.util.strutil import create_path_env_var, safe_shlex_join +logger = logging.getLogger(__name__) + + class PythonNativeCode(Subsystem): """A subsystem which exposes components of the native backend to the python backend.""" @@ -228,10 +232,10 @@ def as_environment(self): ret['PATH'] = create_path_env_var(all_path_entries) all_library_dirs = ( - c_compiler.library_dirs + - c_linker.library_dirs + - cpp_compiler.library_dirs + - cpp_linker.library_dirs) + c_compiler.runtime_library_dirs + + c_linker.runtime_library_dirs + + cpp_compiler.runtime_library_dirs + + cpp_linker.runtime_library_dirs) joined_library_dirs = create_path_env_var(all_library_dirs) dynamic_lib_env_var = plat.resolve_platform_specific({ 'darwin': lambda: 'DYLD_LIBRARY_PATH', @@ -242,22 +246,28 @@ def as_environment(self): all_linking_library_dirs = (c_linker.linking_library_dirs + cpp_linker.linking_library_dirs) ret['LIBRARY_PATH'] = create_path_env_var(all_linking_library_dirs) - all_include_dirs = cpp_compiler.include_dirs + c_compiler.include_dirs - ret['CPATH'] = create_path_env_var(all_include_dirs) + ret['CPATH'] = create_path_env_var(c_compiler.include_dirs) + ret['CPLUS_INCLUDE_PATH'] = create_path_env_var(cpp_compiler.include_dirs) - shared_compile_flags = safe_shlex_join(plat.resolve_platform_specific({ - 'darwin': lambda: [MIN_OSX_VERSION_ARG], - 'linux': lambda: [], + # FIXME: distutils will use CFLAGS to populate LDFLAGS and CXXFLAGS as well if specified -- + # this probably requires a workaround as discused in #5661. + ret['CFLAGS'] = safe_shlex_join(plat.resolve_platform_specific({ + 'darwin': lambda: [MIN_OSX_VERSION_ARG, '-nobuiltininc', '-nostdinc', '-nostdinc++'], + 'linux': lambda: ['-nostdinc', '-nostdinc++'], })) - ret['CFLAGS'] = shared_compile_flags - ret['CXXFLAGS'] = shared_compile_flags + # ret['CFLAGS'] = safe_shlex_join(c_compiler.extra_args) + # ret['CXXFLAGS'] = safe_shlex_join(cpp_compiler.extra_args) ret['CC'] = c_compiler.exe_filename ret['CXX'] = cpp_compiler.exe_filename ret['LDSHARED'] = cpp_linker.exe_filename - all_new_ldflags = cpp_linker.extra_args + plat.resolve_platform_specific( + all_new_ldflags = list(cpp_linker.extra_args) + plat.resolve_platform_specific( self._SHARED_CMDLINE_ARGS) + logger.debug("all_new_ldflags: {}".format(all_new_ldflags)) ret['LDFLAGS'] = safe_shlex_join(all_new_ldflags) + # This avoids decoding errors parsing unicode smart quotes that gcc outputs for fun. + ret['LC_ALL'] = 'C' + return ret diff --git a/src/python/pants/engine/isolated_process.py b/src/python/pants/engine/isolated_process.py index 218321885f0..6af014f31a9 100644 --- a/src/python/pants/engine/isolated_process.py +++ b/src/python/pants/engine/isolated_process.py @@ -8,7 +8,7 @@ from future.utils import binary_type, text_type -from pants.engine.fs import DirectoryDigest +from pants.engine.fs import EMPTY_SNAPSHOT, DirectoryDigest from pants.engine.rules import RootRule, rule from pants.engine.selectors import Select from pants.util.objects import Exactly, TypeCheckError, datatype @@ -68,6 +68,16 @@ def __new__( jdk_home=jdk_home, ) + @classmethod + def create_from_input_snapshot(cls, argv, snapshot, description, **kwargs): + return cls(argv=argv, input_files=snapshot.directory_digest, description=text_type(description), + **kwargs) + + @classmethod + def create_with_empty_snapshot(cls, argv, description, **kwargs): + return cls.create_from_input_snapshot( + argv=argv, snapshot=EMPTY_SNAPSHOT, description=description, **kwargs) + class ExecuteProcessResult(datatype([('stdout', binary_type), ('stderr', binary_type), diff --git a/tests/python/pants_test/backend/native/subsystems/BUILD b/tests/python/pants_test/backend/native/subsystems/BUILD index 2942783eae4..f392f13db92 100644 --- a/tests/python/pants_test/backend/native/subsystems/BUILD +++ b/tests/python/pants_test/backend/native/subsystems/BUILD @@ -4,6 +4,7 @@ python_tests( 'src/python/pants/backend/native/config', 'src/python/pants/backend/native/subsystems', 'src/python/pants/backend/native/subsystems/binaries', + 'src/python/pants/engine:isolated_process', 'src/python/pants/util:contextutil', 'src/python/pants/util:osutil', 'src/python/pants/util:process_handler', diff --git a/tests/python/pants_test/backend/native/subsystems/test_libc_resolution.py b/tests/python/pants_test/backend/native/subsystems/test_libc_resolution.py deleted file mode 100644 index 9b7f15d581c..00000000000 --- a/tests/python/pants_test/backend/native/subsystems/test_libc_resolution.py +++ /dev/null @@ -1,79 +0,0 @@ -# coding=utf-8 -# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import absolute_import, division, print_function, unicode_literals - -from pants.backend.native.config.environment import Platform -from pants.backend.native.subsystems.libc_dev import LibcDev -from pants.backend.native.subsystems.utils.parse_search_dirs import ParseSearchDirs -from pants_test.backend.native.util.platform_utils import platform_specific -from pants_test.subsystem.subsystem_util import global_subsystem_instance, init_subsystems -from pants_test.test_base import TestBase - - -class TestLibcDirectorySearchFailure(TestBase): - - def setUp(self): - init_subsystems([LibcDev], options={ - 'libc': { - 'enable_libc_search': True, - 'libc_dir': '/does/not/exist', - }, - }) - - self.libc = global_subsystem_instance(LibcDev) - self.platform = Platform.create() - - @platform_specific('linux') - def test_libc_search_failure(self): - with self.assertRaises(LibcDev.HostLibcDevResolutionError) as cm: - self.libc.get_libc_dirs(self.platform) - expected_msg = ( - "Could not locate crti.o in directory /does/not/exist provided by the --libc-dir option.") - self.assertEqual(expected_msg, str(cm.exception)) - - @platform_specific('darwin') - def test_libc_search_noop_osx(self): - self.assertEqual([], self.libc.get_libc_dirs(self.platform)) - - -class TestLibcSearchDisabled(TestBase): - - def setUp(self): - init_subsystems([LibcDev], options={ - 'libc': { - 'enable_libc_search': False, - 'libc_dir': '/does/not/exist', - }, - }) - - self.libc = global_subsystem_instance(LibcDev) - self.platform = Platform.create() - - @platform_specific('linux') - def test_libc_disabled_search(self): - self.assertEqual([], self.libc.get_libc_dirs(self.platform)) - - -class TestLibcCompilerSearchFailure(TestBase): - - def setUp(self): - init_subsystems([LibcDev], options={ - 'libc': { - 'enable_libc_search': True, - 'host_compiler': 'this_executable_does_not_exist', - }, - }) - - self.libc = global_subsystem_instance(LibcDev) - self.platform = Platform.create() - - @platform_specific('linux') - def test_libc_compiler_search_failure(self): - with self.assertRaises(ParseSearchDirs.ParseSearchDirsError) as cm: - self.libc.get_libc_dirs(self.platform) - expected_msg = ( - "Process invocation with argv " - "'this_executable_does_not_exist -print-search-dirs' and environment None failed.") - self.assertIn(expected_msg, str(cm.exception)) diff --git a/tests/python/pants_test/backend/native/subsystems/test_native_toolchain.py b/tests/python/pants_test/backend/native/subsystems/test_native_toolchain.py index e7e9fe571c5..32eb4971ef4 100644 --- a/tests/python/pants_test/backend/native/subsystems/test_native_toolchain.py +++ b/tests/python/pants_test/backend/native/subsystems/test_native_toolchain.py @@ -13,14 +13,14 @@ from pants.backend.native.register import rules as native_backend_rules from pants.backend.native.subsystems.binaries.gcc import GCC from pants.backend.native.subsystems.binaries.llvm import LLVM -from pants.backend.native.subsystems.libc_dev import LibcDev from pants.backend.native.subsystems.native_toolchain import NativeToolchain +from pants.engine.isolated_process import create_process_rules from pants.util.contextutil import environment_as, pushd, temporary_dir from pants.util.dirutil import is_executable, safe_open from pants.util.process_handler import subprocess from pants.util.strutil import safe_shlex_join from pants_test.engine.scheduler_test_base import SchedulerTestBase -from pants_test.subsystem.subsystem_util import global_subsystem_instance, init_subsystems +from pants_test.subsystem.subsystem_util import global_subsystem_instance from pants_test.test_base import TestBase @@ -29,15 +29,9 @@ class TestNativeToolchain(TestBase, SchedulerTestBase): def setUp(self): super(TestNativeToolchain, self).setUp() - init_subsystems([LibcDev, NativeToolchain], options={ - 'libc': { - 'enable_libc_search': True, - }, - }) - self.platform = Platform.create() self.toolchain = global_subsystem_instance(NativeToolchain) - self.rules = native_backend_rules() + self.rules = (native_backend_rules() + create_process_rules()) gcc_subsystem = global_subsystem_instance(GCC) self.gcc_version = gcc_subsystem.version() @@ -118,13 +112,13 @@ def _hello_world_source_environment(self, toolchain_type, file_name, contents): yield toolchain def _invoke_compiler(self, compiler, args): - cmd = [compiler.exe_filename] + compiler.extra_args + args + cmd = [compiler.exe_filename] + list(compiler.extra_args) + args return self._invoke_capturing_output( cmd, compiler.as_invocation_environment_dict) def _invoke_linker(self, linker, args): - cmd = [linker.exe_filename] + linker.extra_args + args + cmd = [linker.exe_filename] + list(linker.extra_args) + args return self._invoke_capturing_output( cmd, linker.as_invocation_environment_dict) diff --git a/tests/python/pants_test/backend/python/tasks/util/BUILD b/tests/python/pants_test/backend/python/tasks/util/BUILD index b2e6214c9b6..01538efa779 100644 --- a/tests/python/pants_test/backend/python/tasks/util/BUILD +++ b/tests/python/pants_test/backend/python/tasks/util/BUILD @@ -2,6 +2,7 @@ python_library( dependencies=[ 'src/python/pants/backend/native', 'src/python/pants/backend/python/tasks', + 'src/python/pants/engine:isolated_process', 'tests/python/pants_test/backend/python/tasks:python_task_test_base', 'tests/python/pants_test/engine:scheduler_test_base', ] diff --git a/tests/python/pants_test/backend/python/tasks/util/build_local_dists_test_base.py b/tests/python/pants_test/backend/python/tasks/util/build_local_dists_test_base.py index de57ace976e..7a75dd30ca6 100644 --- a/tests/python/pants_test/backend/python/tasks/util/build_local_dists_test_base.py +++ b/tests/python/pants_test/backend/python/tasks/util/build_local_dists_test_base.py @@ -10,6 +10,7 @@ from pants.backend.native.register import rules as native_backend_rules from pants.backend.python.tasks.build_local_python_distributions import \ BuildLocalPythonDistributions +from pants.engine.isolated_process import create_process_rules from pants.util.collections import assert_single_element from pants_test.backend.python.tasks.python_task_test_base import PythonTaskTestBase from pants_test.engine.scheduler_test_base import SchedulerTestBase @@ -56,7 +57,8 @@ def _all_specified_targets(self): return list(self.target_dict.values()) def _scheduling_context(self, **kwargs): - scheduler = self.mk_scheduler(rules=native_backend_rules()) + all_rules = (native_backend_rules() + create_process_rules()) + scheduler = self.mk_scheduler(rules=all_rules) return self.context(scheduler=scheduler, **kwargs) def _retrieve_single_product_at_target_base(self, product_mapping, target):