Backport Bitcoin bitcoin#11881: Remove Python2 support (dashpay#2224)

* Merge bitcoin#11881: Remove Python2 support

1874058 Make base58 python contrib code work with python3 (Evan Klitzke)
bc6fdf2 Change all python files to use Python3 (John Newbery)

Pull request description:

  Following discussion here: bitcoin#11843 (comment)

  It's easier for maintainers if all python tools/scripts support only a single version of Python. There are only a few scripts that aren't explicitly python3 at this point, so this PR changes those remaining scripts to explicitly require python3.

Tree-SHA512: 5d38eef6e0fc7d8515e23a1f4c75e8b4160fd0fe23cba52a1f41689b114e54a9e503e0724829e8b41982ef98f2d113df80d9e238213b74f09ceaed0344a19e24

* Merge bitcoin#12829: Python3 fixup

f50975b [contrib] fixup symbol-check.py Python3 support (John Newbery)
5de2b18 [contrib] fixup security-check.py Python3 support (John Newbery)

Pull request description:

  security-check.py and symbol-check.py were broken by bitcoin#11881. Fix them.

Tree-SHA512: 86de3d6dc3292b1ae4cc04c2d7d7dbbf39c9270551d7b224b8d8b19e3184c30c897dbf823200403706d06bb405c0decad5cfd690cb2c0312992a235a4ffcf6bf

contrib/devtools/check-doc.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # Copyright (c) 2015 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
@@ -15,29 +15,29 @@
 
 FOLDER_GREP = 'src'
 FOLDER_TEST = 'src/test/'
-CMD_ROOT_DIR = '`git rev-parse --show-toplevel`/%s' % FOLDER_GREP
-CMD_GREP_ARGS = r"egrep -r -I '(map(Multi)?Args(\.count\(|\[)|Get(Bool)?Arg\()\"\-[^\"]+?\"' %s | grep -v '%s'" % (CMD_ROOT_DIR, FOLDER_TEST)
-CMD_GREP_DOCS = r"egrep -r -I 'HelpMessageOpt\(\"\-[^\"=]+?(=|\")' %s" % (CMD_ROOT_DIR)
+CMD_ROOT_DIR = '`git rev-parse --show-toplevel`/{}'.format(FOLDER_GREP)
+CMD_GREP_ARGS = r"egrep -r -I '(map(Multi)?Args(\.count\(|\[)|Get(Bool)?Arg\()\"\-[^\"]+?\"' {} | grep -v '{}'".format(CMD_ROOT_DIR, FOLDER_TEST)
+CMD_GREP_DOCS = r"egrep -r -I 'HelpMessageOpt\(\"\-[^\"=]+?(=|\")' {}".format(CMD_ROOT_DIR)
 REGEX_ARG = re.compile(r'(?:map(?:Multi)?Args(?:\.count\(|\[)|Get(?:Bool)?Arg\()\"(\-[^\"]+?)\"')
 REGEX_DOC = re.compile(r'HelpMessageOpt\(\"(\-[^\"=]+?)(?:=|\")')
 # list unsupported, deprecated and duplicate args as they need no documentation
 SET_DOC_OPTIONAL = set(['-rpcssl', '-benchmark', '-h', '-help', '-socks', '-tor', '-debugnet', '-whitelistalwaysrelay', '-blockminsize'])
 
 def main():
-  used = check_output(CMD_GREP_ARGS, shell=True)
-  docd = check_output(CMD_GREP_DOCS, shell=True)
+  used = check_output(CMD_GREP_ARGS, shell=True, universal_newlines=True)
+  docd = check_output(CMD_GREP_DOCS, shell=True, universal_newlines=True)
 
   args_used = set(re.findall(REGEX_ARG,used))
   args_docd = set(re.findall(REGEX_DOC,docd)).union(SET_DOC_OPTIONAL)
   args_need_doc = args_used.difference(args_docd)
   args_unknown = args_docd.difference(args_used)
 
-  print "Args used        : %s" % len(args_used)
-  print "Args documented  : %s" % len(args_docd)
-  print "Args undocumented: %s" % len(args_need_doc)
-  print args_need_doc
-  print "Args unknown     : %s" % len(args_unknown)
-  print args_unknown
+  print("Args used        : {}".format(len(args_used)))
+  print("Args documented  : {}".format(len(args_docd)))
+  print("Args undocumented: {}".format(len(args_need_doc)))
+  print(args_need_doc)
+  print("Args unknown     : {}".format(len(args_unknown)))
+  print(args_unknown)
 
   exit(len(args_need_doc))
 

contrib/devtools/clang-format-diff.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 #===- clang-format-diff.py - ClangFormat Diff Reformatter ----*- python -*--===#
 #
@@ -69,10 +69,10 @@
 
 import argparse
 import difflib
+import io
 import re
 import string
 import subprocess
-import StringIO
 import sys
 
 
@@ -133,30 +133,33 @@ def main():
           ['-lines', str(start_line) + ':' + str(end_line)])
 
   # Reformat files containing changes in place.
-  for filename, lines in lines_by_file.iteritems():
+  for filename, lines in lines_by_file.items():
     if args.i and args.verbose:
-      print 'Formatting', filename
+      print('Formatting {}'.format(filename))
     command = [binary, filename]
     if args.i:
       command.append('-i')
     if args.sort_includes:
       command.append('-sort-includes')
     command.extend(lines)
     command.extend(['-style=file', '-fallback-style=none'])
-    p = subprocess.Popen(command, stdout=subprocess.PIPE,
-                         stderr=None, stdin=subprocess.PIPE)
+    p = subprocess.Popen(command,
+                         stdout=subprocess.PIPE,
+                         stderr=None,
+                         stdin=subprocess.PIPE,
+                         universal_newlines=True)
     stdout, stderr = p.communicate()
     if p.returncode != 0:
       sys.exit(p.returncode)
 
     if not args.i:
       with open(filename) as f:
         code = f.readlines()
-      formatted_code = StringIO.StringIO(stdout).readlines()
+      formatted_code = io.StringIO(stdout).readlines()
       diff = difflib.unified_diff(code, formatted_code,
                                   filename, filename,
                                   '(before formatting)', '(after formatting)')
-      diff_string = string.join(diff, '')
+      diff_string = ''.join(diff)
       if len(diff_string) > 0:
         sys.stdout.write(diff_string)
 

contrib/devtools/optimize-pngs.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # Copyright (c) 2014-2015 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
@@ -10,7 +10,7 @@
 import sys
 import subprocess
 import hashlib
-from PIL import Image
+from PIL import Image  # pip3 install Pillow
 
 def file_hash(filename):
     '''Return hash of raw file contents'''
@@ -27,7 +27,7 @@ def content_hash(filename):
 pngcrush = 'pngcrush'
 git = 'git'
 folders = ["src/qt/res/movies", "src/qt/res/icons", "src/qt/res/icons/crownium", "src/qt/res/icons/drkblue", "src/qt/res/icons/light", "src/qt/res/icons/light-retro", "src/qt/res/icons/trad", "src/qt/res/images", "src/qt/res/images/crownium", "src/qt/res/images/drkblue", "src/qt/res/images/light", "src/qt/res/images/light-retro", "src/qt/res/images/trad", "share/pixmaps"]
-basePath = subprocess.check_output([git, 'rev-parse', '--show-toplevel']).rstrip('\n')
+basePath = subprocess.check_output([git, 'rev-parse', '--show-toplevel'], universal_newlines=True).rstrip('\n')
 totalSaveBytes = 0
 noHashChange = True
 
@@ -37,42 +37,40 @@ def content_hash(filename):
     for file in os.listdir(absFolder):
         extension = os.path.splitext(file)[1]
         if extension.lower() == '.png':
-            print("optimizing "+file+"..."),
+            print("optimizing {}...".format(file), end =' ')
             file_path = os.path.join(absFolder, file)
             fileMetaMap = {'file' : file, 'osize': os.path.getsize(file_path), 'sha256Old' : file_hash(file_path)}
             fileMetaMap['contentHashPre'] = content_hash(file_path)
 
-            pngCrushOutput = ""
             try:
-                pngCrushOutput = subprocess.check_output(
-                        [pngcrush, "-brute", "-ow", "-rem", "gAMA", "-rem", "cHRM", "-rem", "iCCP", "-rem", "sRGB", "-rem", "alla", "-rem", "text", file_path],
-                        stderr=subprocess.STDOUT).rstrip('\n')
+                subprocess.call([pngcrush, "-brute", "-ow", "-rem", "gAMA", "-rem", "cHRM", "-rem", "iCCP", "-rem", "sRGB", "-rem", "alla", "-rem", "text", file_path],
+                                stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
             except:
-                print "pngcrush is not installed, aborting..."
+                print("pngcrush is not installed, aborting...")
                 sys.exit(0)
 
             #verify
-            if "Not a PNG file" in subprocess.check_output([pngcrush, "-n", "-v", file_path], stderr=subprocess.STDOUT):
-                print "PNG file "+file+" is corrupted after crushing, check out pngcursh version"
+            if "Not a PNG file" in subprocess.check_output([pngcrush, "-n", "-v", file_path], stderr=subprocess.STDOUT, universal_newlines=True):
+                print("PNG file "+file+" is corrupted after crushing, check out pngcursh version")
                 sys.exit(1)
 
             fileMetaMap['sha256New'] = file_hash(file_path)
             fileMetaMap['contentHashPost'] = content_hash(file_path)
 
             if fileMetaMap['contentHashPre'] != fileMetaMap['contentHashPost']:
-                print "Image contents of PNG file "+file+" before and after crushing don't match"
+                print("Image contents of PNG file {} before and after crushing don't match".format(file))
                 sys.exit(1)
 
             fileMetaMap['psize'] = os.path.getsize(file_path)
             outputArray.append(fileMetaMap)
-            print("done\n"),
+            print("done")
 
-print "summary:\n+++++++++++++++++"
+print("summary:\n+++++++++++++++++")
 for fileDict in outputArray:
     oldHash = fileDict['sha256Old']
     newHash = fileDict['sha256New']
     totalSaveBytes += fileDict['osize'] - fileDict['psize']
     noHashChange = noHashChange and (oldHash == newHash)
-    print fileDict['file']+"\n  size diff from: "+str(fileDict['osize'])+" to: "+str(fileDict['psize'])+"\n  old sha256: "+oldHash+"\n  new sha256: "+newHash+"\n"
+    print(fileDict['file']+"\n  size diff from: "+str(fileDict['osize'])+" to: "+str(fileDict['psize'])+"\n  old sha256: "+oldHash+"\n  new sha256: "+newHash+"\n")
 
-print "completed. Checksum stable: "+str(noHashChange)+". Total reduction: "+str(totalSaveBytes)+" bytes"
+print("completed. Checksum stable: "+str(noHashChange)+". Total reduction: "+str(totalSaveBytes)+" bytes")

contrib/devtools/security-check.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # Copyright (c) 2015-2016 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
@@ -8,7 +8,6 @@
 Otherwise the exit status will be 1 and it will log which executables failed which checks.
 Needs `readelf` (for ELF) and `objdump` (for PE).
 '''
-from __future__ import division,print_function,unicode_literals
 import subprocess
 import sys
 import os
@@ -21,38 +20,38 @@ def check_ELF_PIE(executable):
     '''
     Check for position independent executable (PIE), allowing for address space randomization.
     '''
-    p = subprocess.Popen([READELF_CMD, '-h', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+    p = subprocess.Popen([READELF_CMD, '-h', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True)
     (stdout, stderr) = p.communicate()
     if p.returncode:
         raise IOError('Error opening file')
 
     ok = False
-    for line in stdout.split(b'\n'):
+    for line in stdout.splitlines():
         line = line.split()
-        if len(line)>=2 and line[0] == b'Type:' and line[1] == b'DYN':
+        if len(line)>=2 and line[0] == 'Type:' and line[1] == 'DYN':
             ok = True
     return ok
 
 def get_ELF_program_headers(executable):
     '''Return type and flags for ELF program headers'''
-    p = subprocess.Popen([READELF_CMD, '-l', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+    p = subprocess.Popen([READELF_CMD, '-l', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True)
     (stdout, stderr) = p.communicate()
     if p.returncode:
         raise IOError('Error opening file')
     in_headers = False
     count = 0
     headers = []
-    for line in stdout.split(b'\n'):
-        if line.startswith(b'Program Headers:'):
+    for line in stdout.splitlines():
+        if line.startswith('Program Headers:'):
             in_headers = True
-        if line == b'':
+        if line == '':
             in_headers = False
         if in_headers:
             if count == 1: # header line
-                ofs_typ = line.find(b'Type')
-                ofs_offset = line.find(b'Offset')
-                ofs_flags = line.find(b'Flg')
-                ofs_align = line.find(b'Align')
+                ofs_typ = line.find('Type')
+                ofs_offset = line.find('Offset')
+                ofs_flags = line.find('Flg')
+                ofs_align = line.find('Align')
                 if ofs_typ == -1 or ofs_offset == -1 or ofs_flags == -1 or ofs_align  == -1:
                     raise ValueError('Cannot parse elfread -lW output')
             elif count > 1:
@@ -69,9 +68,9 @@ def check_ELF_NX(executable):
     have_wx = False
     have_gnu_stack = False
     for (typ, flags) in get_ELF_program_headers(executable):
-        if typ == b'GNU_STACK':
+        if typ == 'GNU_STACK':
             have_gnu_stack = True
-        if b'W' in flags and b'E' in flags: # section is both writable and executable
+        if 'W' in flags and 'E' in flags: # section is both writable and executable
             have_wx = True
     return have_gnu_stack and not have_wx
 
@@ -88,31 +87,31 @@ def check_ELF_RELRO(executable):
         # However, the dynamic linker need to write to this area so these are RW.
         # Glibc itself takes care of mprotecting this area R after relocations are finished.
         # See also http://permalink.gmane.org/gmane.comp.gnu.binutils/71347
-        if typ == b'GNU_RELRO':
+        if typ == 'GNU_RELRO':
             have_gnu_relro = True
 
     have_bindnow = False
-    p = subprocess.Popen([READELF_CMD, '-d', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+    p = subprocess.Popen([READELF_CMD, '-d', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True)
     (stdout, stderr) = p.communicate()
     if p.returncode:
         raise IOError('Error opening file')
-    for line in stdout.split(b'\n'):
+    for line in stdout.splitlines():
         tokens = line.split()
-        if len(tokens)>1 and tokens[1] == b'(BIND_NOW)' or (len(tokens)>2 and tokens[1] == b'(FLAGS)' and b'BIND_NOW' in tokens[2]):
+        if len(tokens)>1 and tokens[1] == '(BIND_NOW)' or (len(tokens)>2 and tokens[1] == '(FLAGS)' and 'BIND_NOW' in tokens[2]):
             have_bindnow = True
     return have_gnu_relro and have_bindnow
 
 def check_ELF_Canary(executable):
     '''
     Check for use of stack canary
     '''
-    p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+    p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True)
     (stdout, stderr) = p.communicate()
     if p.returncode:
         raise IOError('Error opening file')
     ok = False
-    for line in stdout.split(b'\n'):
-        if b'__stack_chk_fail' in line:
+    for line in stdout.splitlines():
+        if '__stack_chk_fail' in line:
             ok = True
     return ok
 
@@ -122,13 +121,13 @@ def get_PE_dll_characteristics(executable):
     Returns a tuple (arch,bits) where arch is 'i386:x86-64' or 'i386'
     and bits is the DllCharacteristics value.
     '''
-    p = subprocess.Popen([OBJDUMP_CMD, '-x',  executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+    p = subprocess.Popen([OBJDUMP_CMD, '-x',  executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True)
     (stdout, stderr) = p.communicate()
     if p.returncode:
         raise IOError('Error opening file')
     arch = ''
     bits = 0
-    for line in stdout.split('\n'):
+    for line in stdout.splitlines():
         tokens = line.split()
         if len(tokens)>=2 and tokens[0] == 'architecture:':
             arch = tokens[1].rstrip(',')