Python 3 fixes - fix contrib folders problems (pantsbuild#6272)

Most folders are still failing due to the Rust ImportError, but this fixes a lot of their issues.

I'll be submitting a separate PR for porting contrib/python, because it involves major changes resulting from changes in the AST grammar that are better reviewed independently from this.

contrib/avro/tests/python/pants_test/contrib/avro/tasks/test_avro_gen.py

@@ -58,7 +58,7 @@ def test_avro_java_gen(self):
       )
     '''))
 
-    self.create_file(relpath='avro-build/src/avro/schema.avsc', contents=dedent('''
+    self.create_file(relpath='avro-build/src/avro/schema.avsc', mode='w', contents=dedent('''
       {
         "namespace": "",
         "type": "record",
@@ -70,7 +70,7 @@ def test_avro_java_gen(self):
       }
     '''))
 
-    self.create_file(relpath='avro-build/src/avro/record.avdl', contents=dedent('''
+    self.create_file(relpath='avro-build/src/avro/record.avdl', mode='w', contents=dedent('''
       protocol Test {
         void test();
       }

contrib/confluence/src/python/pants/contrib/confluence/util/confluence_util.py

@@ -9,6 +9,9 @@
 import mimetypes
 from builtins import object, str
 from os.path import basename
+from xmlrpc.client import Binary
+from xmlrpc.client import Error as XMLRPCError
+from xmlrpc.client import ServerProxy
 
 from future.moves.urllib.parse import quote_plus
 
@@ -21,10 +24,6 @@
 
 """Code to ease publishing text to Confluence wikis."""
 
-try:
-  from xmlrpc.client import ServerProxy, Error as XMLRPCError, Binary
-except ImportError:
-  from xmlrpclib import ServerProxy, Error as XMLRPCError, Binary
 
 mimetypes.init()
 

contrib/findbugs/tests/python/pants_test/contrib/findbugs/tasks/test_findbugs_integration.py

@@ -79,7 +79,7 @@ def test_max_rank_fail_on_error(self):
 
   def test_exclude(self):
     cmd = ['compile', 'contrib/findbugs/tests/java/org/pantsbuild/contrib/findbugs::']
-    with temporary_file(root_dir=get_buildroot()) as exclude_file:
+    with temporary_file(root_dir=get_buildroot(), binary_mode=False) as exclude_file:
       exclude_file.write(dedent("""\
         <?xml version="1.0" encoding="UTF-8"?>
         <FindBugsFilter>
@@ -102,7 +102,7 @@ def test_exclude(self):
 
   def test_error(self):
     cmd = ['compile', 'contrib/findbugs/tests/java/org/pantsbuild/contrib/findbugs:high']
-    with temporary_file(root_dir=get_buildroot()) as exclude_file:
+    with temporary_file(root_dir=get_buildroot(), binary_mode=False) as exclude_file:
       exclude_file.write(dedent("""\
         <?xml version="1.0" encoding="UTF-8"?>
         <FindBugsFilter>

contrib/go/tests/python/pants_test/contrib/go/subsystems/test_go_distribution.py

@@ -23,8 +23,8 @@ def distribution(self):
   def test_bootstrap(self):
     go_distribution = self.distribution()
     go_cmd = go_distribution.create_go_cmd(cmd='env', args=['GOROOT'])
-    output = go_cmd.check_output()
-    self.assertEqual(go_distribution.goroot, output.strip())
+    output = go_cmd.check_output().decode('utf-8').strip()
+    self.assertEqual(go_distribution.goroot, output)
 
   def assert_no_gopath(self):
     go_distribution = self.distribution()
@@ -39,7 +39,7 @@ def assert_no_gopath(self):
     cmd = [os.path.join(go_distribution.goroot, 'bin', 'go'), 'env', 'GOPATH']
     env = os.environ.copy()
     env.update(go_env)
-    default_gopath = subprocess.check_output(cmd, env=env).strip()
+    default_gopath = subprocess.check_output(cmd, env=env).decode('utf-8').strip()
 
     go_cmd = go_distribution.create_go_cmd(cmd='env', args=['GOPATH'])
 
@@ -48,7 +48,7 @@ def assert_no_gopath(self):
     self.assertEqual(['env', 'GOPATH'], go_cmd.cmdline[1:])
     self.assertRegexpMatches(str(go_cmd),
                              r'^GOROOT=[^ ]+ GOPATH={} .*/go env GOPATH'.format(default_gopath))
-    self.assertEqual(default_gopath, go_cmd.check_output().strip())
+    self.assertEqual(default_gopath, go_cmd.check_output().decode('utf-8').strip())
 
   def test_go_command_no_gopath(self):
     self.assert_no_gopath()

contrib/go/tests/python/pants_test/contrib/go/tasks/test_go_binary_integration.py

@@ -19,4 +19,4 @@ def test_go_crosscompile(self):
             'contrib/go/examples/src/go/hello']
     pants_run = self.run_pants(args, extra_env={"GOOS": "windows"})
     self.assert_success(pants_run)
-    self.assertIn("for MS Windows", subprocess.check_output(["file", output_file]))
+    self.assertIn(b"for MS Windows", subprocess.check_output(["file", output_file]))

contrib/go/tests/python/pants_test/contrib/go/tasks/test_go_buildgen.py

@@ -93,7 +93,7 @@ def test_multiple_remote_roots_failure(self):
       task.execute()
 
   def test_existing_targets_wrong_type(self):
-    self.create_file(relpath='src/go/src/fred/foo.go', contents=dedent("""
+    self.create_file(relpath='src/go/src/fred/foo.go', mode='w', contents=dedent("""
       package main
 
       import "fmt"
@@ -109,7 +109,7 @@ def test_existing_targets_wrong_type(self):
     self.assertEqual(GoTargetGenerator.WrongLocalSourceTargetTypeError, type(exc.exception.cause))
 
   def test_noop_applicable_targets_simple(self):
-    self.create_file(relpath='src/go/src/fred/foo.go', contents=dedent("""
+    self.create_file(relpath='src/go/src/fred/foo.go', mode='w', contents=dedent("""
       package main
 
       import "fmt"
@@ -125,13 +125,13 @@ def test_noop_applicable_targets_simple(self):
     self.assertEqual(expected, context.targets())
 
   def test_noop_applicable_targets_complete_graph(self):
-    self.create_file(relpath='src/go/src/jane/bar.go', contents=dedent("""
+    self.create_file(relpath='src/go/src/jane/bar.go', mode='w', contents=dedent("""
       package jane
 
       var PublicConstant = 42
     """))
     jane = self.make_target('src/go/src/jane', GoLibrary)
-    self.create_file(relpath='src/go/src/fred/foo.go', contents=dedent("""
+    self.create_file(relpath='src/go/src/fred/foo.go', mode='w', contents=dedent("""
       package main
 
       import (
@@ -157,12 +157,12 @@ def stitch_deps_local(self, materialize):
       # We need physical directories on disk for `--materialize` since it does scans.
       self.create_dir('src/go/src')
 
-    self.create_file(relpath='src/go/src/jane/bar.go', contents=dedent("""
+    self.create_file(relpath='src/go/src/jane/bar.go', mode='w', contents=dedent("""
         package jane
 
         var PublicConstant = 42
       """))
-    self.create_file(relpath='src/go/src/fred/foo.go', contents=dedent("""
+    self.create_file(relpath='src/go/src/fred/foo.go', mode='w', contents=dedent("""
         package main
 
         import (
@@ -218,14 +218,14 @@ def stitch_deps_remote(self, remote=True, materialize=False, fail_floating=False
       self.create_dir('3rdparty/go')
       self.create_dir('src/go/src')
 
-    self.create_file(relpath='src/go/src/jane/bar.go', contents=dedent("""
+    self.create_file(relpath='src/go/src/jane/bar.go', mode='w', contents=dedent("""
         package jane
 
         import "pantsbuild.org/fake/prod"
 
         var PublicConstant = prod.DoesNotExistButWeShouldNotCareWhenCheckingDepsAndNotInstalling
       """))
-    self.create_file(relpath='src/go/src/fred/foo.go', contents=dedent("""
+    self.create_file(relpath='src/go/src/fred/foo.go', mode='w', contents=dedent("""
         package main
 
         import (
@@ -318,7 +318,7 @@ def test_issues_2395(self):
     self.add_to_build_file(relpath='3rdparty/go/pantsbuild.org/fake',
                            target='go_remote_library(rev="v4.5.6")')
 
-    self.create_file(relpath='src/go/src/jane/bar.go', contents=dedent("""
+    self.create_file(relpath='src/go/src/jane/bar.go', mode='w', contents=dedent("""
         package jane
 
         import "pantsbuild.org/fake"
@@ -339,12 +339,12 @@ def test_issues_2395(self):
   def test_issues_2616(self):
     self.set_options(remote=False)
 
-    self.create_file(relpath='src/go/src/jane/bar.go', contents=dedent("""
+    self.create_file(relpath='src/go/src/jane/bar.go', mode='w', contents=dedent("""
         package jane
 
         var PublicConstant = 42
       """))
-    self.create_file(relpath='src/go/src/fred/foo.go', contents=dedent("""
+    self.create_file(relpath='src/go/src/fred/foo.go', mode='w', contents=dedent("""
         package main
 
         /*
@@ -379,19 +379,19 @@ def test_issues_2787(self):
 
     self.set_options(remote=False, materialize=False)
 
-    self.create_file(relpath='src/go/src/helper/helper.go', contents=dedent("""
+    self.create_file(relpath='src/go/src/helper/helper.go', mode='w', contents=dedent("""
         package helper
 
         const PublicConstant = 42
       """))
 
-    self.create_file(relpath='src/go/src/lib/lib.go', contents=dedent("""
+    self.create_file(relpath='src/go/src/lib/lib.go', mode='w', contents=dedent("""
         package lib
 
         const privateConstant = 42
       """))
 
-    self.create_file(relpath='src/go/src/lib/lib_test.go', contents=dedent("""
+    self.create_file(relpath='src/go/src/lib/lib_test.go', mode='w', contents=dedent("""
         package lib_test
 
         import (

contrib/go/tests/python/pants_test/contrib/go/tasks/test_go_fetch.py

@@ -27,7 +27,7 @@ def task_type(cls):
 
   def test_get_remote_import_paths(self):
     go_fetch = self.create_task(self.context())
-    self.create_file('src/github.com/u/a/a.go', contents="""
+    self.create_file('src/github.com/u/a/a.go', mode='w', contents="""
       package a
 
       import (
@@ -77,7 +77,7 @@ def _create_package(self, dirpath, name, deps):
     """Creates a Go package inside dirpath named 'name' importing deps."""
     imports = ['import "localzip/{}"'.format(d) for d in deps]
     f = os.path.join(dirpath, '{name}/{name}.go'.format(name=name))
-    self.create_file(f, contents=
+    self.create_file(f, mode='w', contents=
       """package {name}
         {imports}
       """.format(name=name, imports='\n'.join(imports)))
@@ -192,7 +192,7 @@ def test_transitive_download_remote_libs_undeclared_deps(self):
 
   def test_issues_2616(self):
     go_fetch = self.create_task(self.context())
-    self.create_file('src/github.com/u/a/a.go', contents="""
+    self.create_file('src/github.com/u/a/a.go', mode='w', contents="""
       package a
 
       import (
@@ -203,7 +203,7 @@ def test_issues_2616(self):
         "bitbucket.org/u/b"
       )
     """)
-    self.create_file('src/github.com/u/a/b.go', contents="""
+    self.create_file('src/github.com/u/a/b.go', mode='w', contents="""
       package a
 
       /*

contrib/googlejavaformat/tests/python/pants_test/contrib/googlejavaformat/test_googlejavaformat.py

@@ -53,6 +53,7 @@ def task_type(cls):
   def test_googlejavaformat(self):
     javafile = self.create_file(
       relpath='src/java/org/pantsbuild/contrib/googlejavaformat/MyClass.java',
+      mode='w',
       contents=self._BADFORMAT)
     target = self.make_target(
       spec='src/java/org/pantsbuild/contrib/googlejavaformat',
@@ -75,6 +76,7 @@ def task_type(cls):
   def test_lint_badformat(self):
     self.create_file(
       relpath='src/java/org/pantsbuild/contrib/googlejavaformat/MyClass.java',
+      mode='w',
       contents=self._BADFORMAT)
     target = self.make_target(
       spec='src/java/org/pantsbuild/contrib/googlejavaformat',
@@ -92,6 +94,7 @@ def test_lint_badformat(self):
   def test_lint_goodformat(self):
     self.create_file(
       relpath='src/java/org/pantsbuild/contrib/googlejavaformat/MyClass.java',
+      mode='w',
       contents=self._GOODFORMAT)
     target = self.make_target(
       spec='src/java/org/pantsbuild/contrib/googlejavaformat',

contrib/node/tests/python/pants_test/contrib/node/subsystems/test_node_distribution.py

@@ -21,28 +21,28 @@ def setUp(self):
 
   def test_bootstrap(self):
     node_cmd = self.distribution.node_command(args=['--version'])
-    output = node_cmd.check_output()
-    self.assertEqual(self.distribution.version(), output.strip())
+    output = node_cmd.check_output().decode('utf-8').strip()
+    self.assertEqual(self.distribution.version(), output)
 
   def test_node(self):
     node_command = self.distribution.node_command(args=['--interactive'])  # Force a REPL session.
     repl = node_command.run(stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
-    out, err = repl.communicate('console.log("Hello World!")')
-    self.assertEqual('', err)
+    out, err = repl.communicate(b'console.log("Hello World!")')
+    self.assertEqual(b'', err)
     self.assertEqual(0, repl.returncode)
 
     for line in out.splitlines():
-      if line.endswith('Hello World!'):
+      if line.endswith(b'Hello World!'):
         break
     else:
       self.fail('Did not find the expected "Hello World!" in the REPL session '
-                'output:\n{}'.format(out))
+                'output:\n{}'.format(out.decode('utf-8')))
 
   def test_npm(self):
     npm_version_flag = self.distribution.get_package_manager('npm').run_command(
       args=['--version'])
-    raw_version = npm_version_flag.check_output().strip()
+    raw_version = npm_version_flag.check_output().decode('utf-8').strip()
 
     npm_version_cmd = self.distribution.get_package_manager('npm').run_command(
       args=['version', '--json'])
@@ -54,7 +54,7 @@ def test_npm(self):
   def test_yarnpkg(self):
     yarnpkg_version_command = self.distribution.get_package_manager('yarn').run_command(
       args=['--version'])
-    yarnpkg_version = yarnpkg_version_command.check_output().strip()
+    yarnpkg_version = yarnpkg_version_command.check_output().decode('utf-8').strip()
     yarnpkg_versions_command = self.distribution.get_package_manager('yarn').run_command(
       args=['versions', '--json'])
     yarnpkg_versions = json.loads(yarnpkg_versions_command.check_output())
@@ -67,7 +67,7 @@ def test_node_command_path_injection(self):
 
     # Test the case in which we do not pass in env,
     # which should fall back to env=os.environ.copy()
-    injected_paths = node_path_cmd.check_output().strip().split(os.pathsep)
+    injected_paths = node_path_cmd.check_output().decode('utf-8').strip().split(os.pathsep)
     self.assertEqual(node_bin_path, injected_paths[0])
 
   def test_node_command_path_injection_with_overrided_path(self):
@@ -76,7 +76,7 @@ def test_node_command_path_injection_with_overrided_path(self):
     node_bin_path = self.distribution._install_node()
     injected_paths = node_path_cmd.check_output(
       env={'PATH': '/test/path'}
-    ).strip().split(os.pathsep)
+    ).decode('utf-8').strip().split(os.pathsep)
     self.assertEqual(node_bin_path, injected_paths[0])
     self.assertListEqual([node_bin_path, '/test/path'], injected_paths)
 
@@ -86,5 +86,5 @@ def test_node_command_path_injection_with_empty_path(self):
     node_bin_path = self.distribution._install_node()
     injected_paths = node_path_cmd.check_output(
       env={'PATH': ''}
-    ).strip().split(os.pathsep)
+    ).decode('utf-8').strip().split(os.pathsep)
     self.assertListEqual([node_bin_path, ''], injected_paths)

contrib/node/tests/python/pants_test/contrib/node/tasks/test_node_build.py

@@ -102,6 +102,7 @@ def test_node_module_no_artifacts(self):
   def test_run_build_script(self):
     package_json_file = self.create_file(
       'src/node/build_test/package.json',
+      mode='w',
       contents=dedent("""
         {
           "scripts": {
@@ -131,7 +132,7 @@ def test_run_build_script(self):
 
   def test_run_non_existing_script(self):
     package_json_file = self.create_file(
-      'src/node/build_test/package.json', contents='{}')
+      'src/node/build_test/package.json', mode='w', contents='{}')
     build_script = 'my_non_existing_build_scirpt'
     target = self.make_target(
       spec='src/node/build_test',
@@ -144,6 +145,7 @@ def test_run_non_existing_script(self):
   def test_run_no_output_dir(self):
     package_json_file = self.create_file(
       'src/node/build_test/package.json',
+      mode='w',
       contents=dedent("""
         {
           "scripts": {

contrib/node/tests/python/pants_test/contrib/node/tasks/test_node_bundle_integration.py

@@ -136,7 +136,7 @@ def test_no_bundle_for_node_module(self):
   def _extract_archive(self, archive_path):
     with temporary_dir() as temp_dir:
       _, extension = os.path.splitext(archive_path)
-      print (extension)
+      print(extension)
       if extension == '.jar':
         extraction_archiver = create_archiver('zip')
       else:

contrib/node/tests/python/pants_test/contrib/node/tasks/test_node_lint_integration.py

@@ -20,7 +20,7 @@ def test_lint_success_with_target_level_ignore(self):
     path = 'contrib/node/examples/src/node/javascriptstyle-empty/index.js'
     content = 'const console = require(\'console\');\nconsole.log("Double Quotes");\n'
 
-    with self.temporary_file_content(path, content):
+    with self.temporary_file_content(path, content, binary_mode=False):
       command = ['lint',
                  'contrib/node/examples/src/node/javascriptstyle-empty']
       pants_run = self.run_pants(command=command)
@@ -31,7 +31,7 @@ def test_lint_failure_without_target_level_ignore(self):
     path = 'contrib/node/examples/src/node/javascriptstyle-empty/not_ignored_index.js'
     content = 'const console = require(\'console\');\nconsole.log("Double Quotes");\n'
 
-    with self.temporary_file_content(path, content):
+    with self.temporary_file_content(path, content, binary_mode=False):
       command = ['lint',
                  'contrib/node/examples/src/node/javascriptstyle-empty']
       pants_run = self.run_pants(command=command)

contrib/node/tests/python/pants_test/contrib/node/tasks/test_node_repl_integration.py

@@ -57,7 +57,7 @@ def test_run_repl_multiple_targets(self):
         var calc = new Calculator(0);
         calc.add(1);
         console.log('React: ' + reactElem + ', Calc + 1: ' + calc.number);
-      """)
+      """).encode('utf-8')
     pants_run = self.run_pants(command=command, stdin_data=program)
 
     self.assert_success(pants_run)