Skip to content

Commit c306f32

Browse files
author
Stu Hood
authored
Zinc 1.0.0 upgrade: Python portion (#4729)
### Problem (this change depends on #4728) As described on #4728, we currently need internal knowledge of zinc analysis in order to make it portable, and to extract products that are used by other tasks. Since the zinc analysis format is changing significantly (moving to protobuf, gaining/losing fields, etc), we need to stop parsing it. ### Solution Incorporates the version of the zinc wrapper published in #4728, and uses it in a `analysis.zinc` task to generate the `product_deps_by_src` and `classes_by_source` products from a `zinc_analysis` product published by zinc. Expands the extracted `Zinc` subsystem from #4720 to encapsulate compile dependency calculation, and expose the `FingerprintStrategy` and `DependencyContext` that `compile.zinc` was using. ### Result Zinc analysis is now a black box from the perspective of the pants python code. Also, the `compile.jvm-dep-check` task moved to the `lint` goal, and gained the unused dependency checks that used to be inlined in `compile.zinc` (and which would have prevented separation of the analysis extraction task). Fixes #4477, fixes #4513, fixes #4185, fixes #5444, and hopefully fixes a few other odd incremental compilation bugs.
1 parent c7707da commit c306f32

File tree

69 files changed

+918
-2372
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

69 files changed

+918
-2372
lines changed

build-support/ivy/ivysettings.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ Licensed under the Apache License, Version 2.0 (see LICENSE).
1010
<property name="m2.repo.relpath" value="[organisation]/[module]/[revision]"/>
1111
<property name="m2.repo.pom" value="${m2.repo.relpath}/[module]-[revision].pom"/>
1212
<property name="m2.repo.artifact"
13-
value="${m2.repo.relpath}/[artifact](-[classifier])-[revision].[ext]"/>
13+
value="${m2.repo.relpath}/[artifact]-[revision](-[classifier]).[ext]"/>
1414
<property name="m2.repo.dir" value="${user.home}/.m2/repository" override="false"/>
1515
<property name="kythe.artifact" value="[organisation]/[artifact]/[revision]/[artifact]-[revision].[ext]"/>
1616
<!-- for retrieving jacoco snapshot, remove when a version containing cli is released -->

contrib/scrooge/tests/python/pants_test/contrib/scrooge/tasks/test_scrooge_gen_integration.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ def run_pants(self, command, config=None, stdin_data=None, extra_env=None, **kwa
2020
'pythonpath': ["%(buildroot)s/contrib/scrooge/src/python"],
2121
'backend_packages': ["pants.backend.codegen", "pants.backend.jvm", "pants.contrib.scrooge"]
2222
},
23-
'scala-platform': { 'version': '2.11' },
23+
'scala': { 'version': '2.11' },
2424
'gen.scrooge': {
2525
'service_deps': {
2626
'java': [

examples/src/scala/org/pantsbuild/example/README.md

+3-3
Original file line numberDiff line numberDiff line change
@@ -56,11 +56,11 @@ Scala Version
5656
-------------
5757

5858
You can override the default version of the entire Scala toolchain with the single
59-
`--scala-platform-version` option. You can set that option to one of the supported Scala versions
59+
`--scala-version` option. You can set that option to one of the supported Scala versions
6060
(currently "2.10" or "2.11"), or to the special value "custom".
6161

62-
If you choose a custom version, you must use the `--scala-platform-runtime-spec`,
63-
`--scala-platform-repl-spec` and `--scala-platform-suffix-version` options to provide
62+
If you choose a custom version, you must use the `--scala-runtime-spec`,
63+
`--scala-repl-spec` and `--scala-suffix-version` options to provide
6464
information about your custom Scala version. The first two of these default to the target
6565
addresses `//:scala-library` and `//:scala-repl` respectively, so you can simply define those
6666
targets (in the root `BUILD.tools` file by convention) to point to the relevant JARs.

pants.ini

+1-3
Original file line numberDiff line numberDiff line change
@@ -180,8 +180,6 @@ exclude_patterns: [
180180
[compile.zinc]
181181
jvm_options: [
182182
'-Xmx4g', '-XX:+UseConcMarkSweepGC', '-XX:ParallelGCThreads=4',
183-
# bigger cache size for our big projects (default is just 5)
184-
'-Dzinc.analysis.cache.limit=1000',
185183
]
186184

187185
args: [
@@ -280,7 +278,7 @@ skip: True
280278
[pycheck-context-manager]
281279
skip: True
282280

283-
[scala-platform]
281+
[scala]
284282
version: 2.11
285283

286284

src/python/pants/backend/jvm/register.py

+5-1
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
from pants.backend.jvm.targets.scala_library import ScalaLibrary
3131
from pants.backend.jvm.targets.scalac_plugin import ScalacPlugin
3232
from pants.backend.jvm.targets.unpacked_jars import UnpackedJars
33+
from pants.backend.jvm.tasks.analysis_extraction import AnalysisExtraction
3334
from pants.backend.jvm.tasks.benchmark_run import BenchmarkRun
3435
from pants.backend.jvm.tasks.binary_create import BinaryCreate
3536
from pants.backend.jvm.tasks.bootstrap_jvm_tools import BootstrapJvmTools
@@ -161,6 +162,9 @@ def register_goals():
161162
task(name='zinc', action=ZincCompile).install('compile')
162163
task(name='javac', action=JavacCompile).install('compile')
163164

165+
# Analysis extraction.
166+
task(name='zinc', action=AnalysisExtraction).install('analysis')
167+
164168
# Dependency resolution.
165169
task(name='ivy', action=IvyResolve).install('resolve', first=True)
166170
task(name='coursier', action=CoursierResolve).install('resolve')
@@ -173,7 +177,6 @@ def register_goals():
173177
task(name='services', action=PrepareServices).install('resources')
174178

175179
task(name='export-classpath', action=RuntimeClasspathPublisher).install()
176-
task(name='jvm-dep-check', action=JvmDependencyCheck).install('compile')
177180

178181
task(name='jvm', action=JvmDependencyUsage).install('dep-usage')
179182

@@ -210,6 +213,7 @@ def register_goals():
210213
task(name='scalafmt', action=ScalaFmtCheckFormat, serialize=False).install('lint')
211214
task(name='scalastyle', action=Scalastyle, serialize=False).install('lint')
212215
task(name='checkstyle', action=Checkstyle, serialize=False).install('lint')
216+
task(name='jvm-dep-check', action=JvmDependencyCheck, serialize=False).install('lint')
213217

214218
# Formatting.
215219
task(name='scalafmt', action=ScalaFmtFormat, serialize=False).install('fmt')

src/python/pants/backend/jvm/subsystems/BUILD

+16
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,25 @@
11
# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
22
# Licensed under the Apache License, Version 2.0 (see LICENSE).
33

4+
python_library(
5+
name = 'dependency_context',
6+
sources = ['dependency_context.py'],
7+
dependencies = [
8+
':java',
9+
':scala_platform',
10+
'src/python/pants/backend/codegen/thrift/java',
11+
'src/python/pants/base:fingerprint_strategy',
12+
'src/python/pants/build_graph',
13+
'src/python/pants/java/jar',
14+
],
15+
)
16+
417
python_library(
518
name = 'jvm_tool_mixin',
619
sources = ['jvm_tool_mixin.py'],
720
dependencies = [
821
'src/python/pants/base:exceptions',
22+
'src/python/pants/java/distribution',
923
'src/python/pants/option',
1024
],
1125
)
@@ -117,7 +131,9 @@ python_library(
117131
name = 'zinc',
118132
sources = ['zinc.py'],
119133
dependencies = [
134+
':dependency_context',
120135
':jvm_tool_mixin',
136+
':shader',
121137
'src/python/pants/subsystem',
122138
]
123139
)
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
# coding=utf-8
2+
# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
3+
# Licensed under the Apache License, Version 2.0 (see LICENSE).
4+
5+
from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
6+
unicode_literals, with_statement)
7+
8+
import hashlib
9+
10+
from pants.backend.codegen.protobuf.java.java_protobuf_library import JavaProtobufLibrary
11+
from pants.backend.codegen.thrift.java.java_thrift_library import JavaThriftLibrary
12+
from pants.backend.jvm.subsystems.java import Java
13+
from pants.backend.jvm.subsystems.scala_platform import ScalaPlatform
14+
from pants.backend.jvm.targets.annotation_processor import AnnotationProcessor
15+
from pants.backend.jvm.targets.jar_library import JarLibrary
16+
from pants.backend.jvm.targets.javac_plugin import JavacPlugin
17+
from pants.backend.jvm.targets.scalac_plugin import ScalacPlugin
18+
from pants.base.fingerprint_strategy import FingerprintStrategy
19+
from pants.build_graph.aliased_target import AliasTarget
20+
from pants.build_graph.resources import Resources
21+
from pants.build_graph.target import Target
22+
from pants.build_graph.target_scopes import Scopes
23+
from pants.subsystem.subsystem import Subsystem
24+
25+
26+
class SyntheticTargetNotFound(Exception):
27+
"""Exports were resolved for a thrift target which hasn't had a synthetic target generated yet."""
28+
29+
30+
class DependencyContext(Subsystem):
31+
"""Implements calculating `exports` and exception (compiler-plugin) aware dependencies.
32+
33+
This is a subsystem because in future the compiler plugin types should be injected
34+
via subsystem or option dependencies rather than declared statically.
35+
"""
36+
37+
options_scope = 'jvm-dependency-context'
38+
39+
target_closure_kwargs = dict(include_scopes=Scopes.JVM_COMPILE_SCOPES, respect_intransitive=True)
40+
compiler_plugin_types = (AnnotationProcessor, JavacPlugin, ScalacPlugin)
41+
alias_types = (AliasTarget, Target)
42+
codegen_types = (JavaThriftLibrary, JavaProtobufLibrary)
43+
44+
@classmethod
45+
def subsystem_dependencies(cls):
46+
return super(DependencyContext, cls).subsystem_dependencies() + (Java, ScalaPlatform)
47+
48+
def all_dependencies(self, target):
49+
"""All transitive dependencies of the context's target."""
50+
for dep in target.closure(bfs=True, **self.target_closure_kwargs):
51+
yield dep
52+
53+
def create_fingerprint_strategy(self, classpath_products):
54+
return ResolvedJarAwareFingerprintStrategy(classpath_products, self)
55+
56+
def defaulted_property(self, target, selector):
57+
"""Computes a language property setting for the given JvmTarget.
58+
59+
:param selector A function that takes a target or platform and returns the boolean value of the
60+
property for that target or platform, or None if that target or platform does
61+
not directly define the property.
62+
63+
If the target does not override the language property, returns true iff the property
64+
is true for any of the matched languages for the target.
65+
"""
66+
if selector(target) is not None:
67+
return selector(target)
68+
69+
prop = False
70+
if target.has_sources('.java'):
71+
prop |= selector(Java.global_instance())
72+
if target.has_sources('.scala'):
73+
prop |= selector(ScalaPlatform.global_instance())
74+
return prop
75+
76+
77+
class ResolvedJarAwareFingerprintStrategy(FingerprintStrategy):
78+
"""Task fingerprint strategy that also includes the resolved coordinates of dependent jars."""
79+
80+
def __init__(self, classpath_products, dep_context):
81+
super(ResolvedJarAwareFingerprintStrategy, self).__init__()
82+
self._classpath_products = classpath_products
83+
self._dep_context = dep_context
84+
85+
def compute_fingerprint(self, target):
86+
if isinstance(target, Resources):
87+
# Just do nothing, this kind of dependency shouldn't affect result's hash.
88+
return None
89+
90+
hasher = hashlib.sha1()
91+
hasher.update(target.payload.fingerprint())
92+
if isinstance(target, JarLibrary):
93+
# NB: Collects only the jars for the current jar_library, and hashes them to ensure that both
94+
# the resolved coordinates, and the requested coordinates are used. This ensures that if a
95+
# source file depends on a library with source compatible but binary incompatible signature
96+
# changes between versions, that you won't get runtime errors due to using an artifact built
97+
# against a binary incompatible version resolved for a previous compile.
98+
classpath_entries = self._classpath_products.get_artifact_classpath_entries_for_targets(
99+
[target])
100+
for _, entry in classpath_entries:
101+
hasher.update(str(entry.coordinate))
102+
return hasher.hexdigest()
103+
104+
def direct(self, target):
105+
return self._dep_context.defaulted_property(target, lambda x: x.strict_deps)
106+
107+
def dependencies(self, target):
108+
if self.direct(target):
109+
return target.strict_dependencies(self._dep_context)
110+
return super(ResolvedJarAwareFingerprintStrategy, self).dependencies(target)
111+
112+
def __hash__(self):
113+
# NB: FingerprintStrategy requires a useful override of eq/hash.
114+
return hash(type(self))
115+
116+
def __eq__(self, other):
117+
# NB: See __hash__.
118+
return type(self) == type(other)

src/python/pants/backend/jvm/subsystems/java.py

+8
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,14 @@ def register_options(cls, register):
3939
help='Java compiler to use. If unspecified, we use the compiler '
4040
'embedded in the Java distribution we run on.')
4141

42+
register('--javac-plugins', advanced=True, type=list, fingerprint=True,
43+
help='Use these javac plugins.')
44+
register('--javac-plugin-args', advanced=True, type=dict, default={}, fingerprint=True,
45+
help='Map from javac plugin name to list of arguments for that plugin.')
46+
cls.register_jvm_tool(register, 'javac-plugin-dep', classpath=[],
47+
help='Search for javac plugins here, as well as in any '
48+
'explicit dependencies.')
49+
4250
def injectables(self, build_graph):
4351
tools_jar_address = Address.parse(self._tools_jar_spec)
4452
if not build_graph.contains_address(tools_jar_address):

src/python/pants/backend/jvm/subsystems/jvm_platform.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ def register_options(cls, register):
5757
help='Compile settings that can be referred to by name in jvm_targets.')
5858
register('--default-platform', advanced=True, type=str, default=None, fingerprint=True,
5959
help='Name of the default platform to use if none are specified.')
60-
register('--compiler', choices=['zinc', 'javac'], default='zinc',
60+
register('--compiler', advanced=True, choices=['zinc', 'javac'], default='zinc', fingerprint=True,
6161
help='Java compiler implementation to use.')
6262

6363
@classmethod

src/python/pants/backend/jvm/subsystems/jvm_tool_mixin.py

+19
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
from textwrap import dedent
1010

1111
from pants.base.exceptions import TaskError
12+
from pants.java.distribution.distribution import DistributionLocator
1213
from pants.option.custom_types import target_option
1314

1415

@@ -51,6 +52,10 @@ def is_default(self, options):
5152

5253
_jvm_tools = [] # List of JvmTool objects.
5354

55+
@classmethod
56+
def subsystem_dependencies(cls):
57+
return super(JvmToolMixin, cls).subsystem_dependencies() + (DistributionLocator,)
58+
5459
@classmethod
5560
def get_jvm_options_default(cls, bootstrap_option_values):
5661
"""Subclasses may override to provide different defaults for their JVM options.
@@ -158,6 +163,20 @@ def reset_registered_tools():
158163
"""Needed only for test isolation."""
159164
JvmToolMixin._jvm_tools = []
160165

166+
def set_distribution(self, minimum_version=None, maximum_version=None, jdk=False):
167+
try:
168+
self._dist = DistributionLocator.cached(minimum_version=minimum_version,
169+
maximum_version=maximum_version, jdk=jdk)
170+
except DistributionLocator.Error as e:
171+
raise TaskError(e)
172+
173+
@property
174+
def dist(self):
175+
if getattr(self, '_dist', None) is None:
176+
# Use default until told otherwise.
177+
self.set_distribution()
178+
return self._dist
179+
161180
@classmethod
162181
def tool_jar_from_products(cls, products, key, scope):
163182
"""Get the jar for the tool previously registered under key in the given scope.

src/python/pants/backend/jvm/subsystems/scala_platform.py

+14-2
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,10 @@ class ScalaPlatform(JvmToolMixin, ZincLanguageMixin, InjectablesMixin, Subsystem
3939
4040
:API: public
4141
"""
42-
options_scope = 'scala-platform'
42+
options_scope = 'scala'
43+
44+
deprecated_options_scope = 'scala-platform'
45+
deprecated_options_scope_removal_version = '1.9.0.dev0'
4346

4447
@classmethod
4548
def _create_jardep(cls, name, version):
@@ -88,6 +91,15 @@ def register_style_tool(version):
8891
classpath=[scala_style_jar])
8992

9093
super(ScalaPlatform, cls).register_options(register)
94+
95+
register('--scalac-plugins', advanced=True, type=list, fingerprint=True,
96+
help='Use these scalac plugins.')
97+
register('--scalac-plugin-args', advanced=True, type=dict, default={}, fingerprint=True,
98+
help='Map from scalac plugin name to list of arguments for that plugin.')
99+
cls.register_jvm_tool(register, 'scalac-plugin-dep', classpath=[],
100+
help='Search for scalac plugins here, as well as in any '
101+
'explicit dependencies.')
102+
91103
register('--version', advanced=True, default='2.12',
92104
choices=['2.10', '2.11', '2.12', 'custom'], fingerprint=True,
93105
help='The scala platform version. If --version=custom, the targets '
@@ -153,7 +165,7 @@ def suffix_version(self, name):
153165
raise RuntimeError('Suffix version must be specified if using a custom scala version. '
154166
'Suffix version is used for bootstrapping jars. If a custom '
155167
'scala version is not specified, then the version specified in '
156-
'--scala-platform-suffix-version is used. For example for Scala '
168+
'--scala-suffix-version is used. For example for Scala '
157169
'2.10.7 you would use the suffix version "2.10".')
158170

159171
elif name.endswith(self.version):

0 commit comments

Comments
 (0)