Merge branch 'beta' into dev

.github/workflows/ci.yml

@@ -27,7 +27,7 @@ jobs:
         key: ${{ matrix.os }}-cache-pip
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 
@@ -59,33 +59,9 @@ jobs:
           binutils-s390x-linux-gnu \
           binutils-sparc64-linux-gnu \
           gcc-multilib \
-          libc6-dbg
+          libc6-dbg \
+          elfutils
       
-    - name: Cache for elfutils
-      uses: actions/cache@v1
-      id: cache-elfutils
-      with:
-        path: ~/.cache/elfutils
-        key: ${{ matrix.os }}-cache-elfutils
-
-    # Install newer elfutils version due to regression in 0.176 available in focal.
-    - name: Install newer elfutils
-      env:
-        ELFUTILS_VERSION: 0.181
-      run: |
-        if [[ ! -d ~/.cache/elfutils/elfutils-${ELFUTILS_VERSION} ]]
-        then
-          wget -O /tmp/elfutils-${ELFUTILS_VERSION}.tar.bz2 https://sourceware.org/elfutils/ftp/${ELFUTILS_VERSION}/elfutils-${ELFUTILS_VERSION}.tar.bz2
-          mkdir -p ~/.cache/elfutils && cd ~/.cache/elfutils
-          tar -xf /tmp/elfutils-${ELFUTILS_VERSION}.tar.bz2
-          cd elfutils-${ELFUTILS_VERSION}
-          LDFLAGS=-Wl,-rpath=/usr/local/lib,--enable-new-dtags ./configure --disable-libdebuginfod --disable-debuginfod && make && sudo make install
-        else
-          cd ~/.cache/elfutils/elfutils-${ELFUTILS_VERSION}
-          sudo make install
-        fi
-        eu-unstrip --version
-
     - name: Install RPyC for GDB
       run: |
         sudo apt-get install -y python3-pip

CHANGELOG.md

@@ -86,6 +86,8 @@ The table below shows which release corresponds to each branch, and what date th
 - [#2033][2033] Quote file and core path in generated GDB script
 - [#2035][2035] Change Buffer's parent class to object
 - [#2037][2037] Allow SSH tunnel to be treated like a TCP socket (with 'raw=True')
+- [#2123][2123] Fix ROP without a writeable cache directory
+- [#2124][2124] Fix `tube.recvpred()` timeout argument
 
 [1975]: https://github.com/Gallopsled/pwntools/pull/1975
 [1979]: https://github.com/Gallopsled/pwntools/pull/1979
@@ -95,6 +97,8 @@ The table below shows which release corresponds to each branch, and what date th
 [2033]: https://github.com/Gallopsled/pwntools/pull/2033
 [2035]: https://github.com/Gallopsled/pwntools/pull/2035
 [2037]: https://github.com/Gallopsled/pwntools/pull/2037
+[2123]: https://github.com/Gallopsled/pwntools/pull/2123
+[2124]: https://github.com/Gallopsled/pwntools/pull/2124
 
 ## 4.8.0 (`stable`)
 

pwnlib/asm.py

@@ -136,8 +136,23 @@ def print_binutils_instructions(util, context):
 %(instructions)s
 """.strip() % locals())
 
+
+def check_binutils_version(util):
+    if util_versions[util]:
+        return util_versions[util]
+    result = subprocess.check_output([util, '--version','/dev/null'],
+                                     stderr=subprocess.STDOUT, universal_newlines=True)
+    if 'clang' in result:
+        log.warn_once('Your binutils is clang-based and may not work!\n'
+            'Try installing with: https://docs.pwntools.com/en/stable/install/binutils.html\n'
+            'Reported version: %r', result.strip())
+    version = re.search(r' (\d+\.\d+)', result).group(1)
+    util_versions[util] = version = tuple(map(int, version.split('.')))
+    return version
+
+
 @LocalContext
-def which_binutils(util):
+def which_binutils(util, check_version=False):
     """
     Finds a binutils in the PATH somewhere.
     Expects that the utility is prefixed with the architecture name.
@@ -204,17 +219,23 @@ def which_binutils(util):
 
             for pattern in patterns:
                 for dir in environ['PATH'].split(':'):
-                    res = sorted(glob(path.join(dir, pattern)))
-                    if res:
-                        return res[0]
+                    for res in sorted(glob(path.join(dir, pattern))):
+                        if check_version:
+                            ver = check_binutils_version(res)
+                            return res, ver
+                        return res
 
     # No dice!
     print_binutils_instructions(util, context)
 
-checked_assembler_version = defaultdict(lambda: False)
+util_versions = defaultdict(tuple)
 
 def _assembler():
-    gas = which_binutils('as')
+    gas, version = which_binutils('as', check_version=True)
+    if version < (2, 19):
+        log.warn_once('Your binutils version is too old and may not work!\n'
+            'Try updating with: https://docs.pwntools.com/en/stable/install/binutils.html\n'
+            'Reported version: %r', version)
 
     E = {
         'big':    '-EB',
@@ -246,25 +267,10 @@ def _assembler():
 
     assembler = assemblers.get(context.arch, [gas])
 
-    if not checked_assembler_version[gas]:
-        checked_assembler_version[gas] = True
-        result = subprocess.check_output([gas, '--version','/dev/null'],
-                                         stderr=subprocess.STDOUT, universal_newlines=True)
-        version = re.search(r' (\d+\.\d+)', result).group(1)
-        if 'clang' in result:
-            log.warn_once('Your binutils is clang version and may not work!\n'
-                'Try install with: https://docs.pwntools.com/en/stable/install/binutils.html\n'
-                'Reported Version: %r', result.strip())
-        elif version < '2.19':
-            log.warn_once('Your binutils version is too old and may not work!\n'
-                'Try updating with: https://docs.pwntools.com/en/stable/install/binutils.html\n'
-                'Reported Version: %r', result.strip())
-
-
     return assembler
 
 def _linker():
-    ld  = [which_binutils('ld')]
+    ld, _ = which_binutils('ld', check_version=True)
     bfd = ['--oformat=' + _bfdname()]
 
     E = {
@@ -276,7 +282,16 @@ def _linker():
         'i386': ['-m', 'elf_i386'],
     }.get(context.arch, [])
 
-    return ld + bfd + [E] + arguments
+    return [ld] + bfd + [E] + arguments
+
+
+def _execstack(linker):
+    ldflags = ['-z', 'execstack']
+    version = util_versions[linker[0]]
+    if version >= (2, 39):
+        return ldflags + ['--no-warn-execstack', '--no-warn-rwx-segments']
+    return ldflags
+
 
 def _objcopy():
     return [which_binutils('objcopy')]
@@ -595,7 +610,7 @@ def make_elf(data,
 
         _run(assembler + ['-o', step2, step1])
 
-        linker_options = ['-z', 'execstack']
+        linker_options = _execstack(linker)
         if vma is not None:
             linker_options += ['--section-start=.shellcode=%#x' % vma,
                                '--entry=%#x' % vma]
@@ -689,7 +704,7 @@ def asm(shellcode, vma = 0, extract = True, shared = False):
             shutil.copy(step2, step3)
 
         if vma or not extract:
-            ldflags = ['-z', 'execstack', '-o', step3, step2]
+            ldflags = _execstack(linker) + ['-o', step3, step2]
             if vma:
                 ldflags += ['--section-start=.shellcode=%#x' % vma,
                             '--entry=%#x' % vma]
@@ -771,7 +786,7 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
         >>> print(disasm(unhex('4ff00500'), arch = 'thumb', bits=32))
            0:   f04f 0005       mov.w   r0, #5
         >>> print(disasm(unhex('656664676665400F18A4000000000051'), byte=0, arch='amd64'))
-           0:   gs data16 fs data16 rex nop/reserved BYTE PTR gs:[eax+eax*1+0x0]
+           0:   gs data16 fs rex nop WORD PTR gs:[eax+eax*1+0x0]
            f:   push   rcx
         >>> print(disasm(unhex('01000000'), arch='sparc64'))
            0:   01 00 00 00     nop

pwnlib/commandline/libcdb.py

@@ -4,7 +4,6 @@
 from __future__ import print_function
 
 import re
-import requests
 import shutil
 import sys
 
@@ -138,6 +137,7 @@
 common_symbols = ['dup2', 'printf', 'puts', 'read', 'system', 'write']
 
 def find_libc(params):
+    import requests
     url    = "https://libc.rip/api/find"
     result = requests.post(url, json=params, timeout=20)
     log.debug('Request: %s', params)

pwnlib/elf/corefile.py

@@ -445,7 +445,7 @@ class Corefile(ELF):
         will be loaded.
 
         >>> process('bash').corefile.libc # doctest: +ELLIPSIS
-        Mapping('/.../libc-....so', start=0x..., stop=0x..., size=0x..., flags=..., page_offset=...)
+        Mapping('.../libc...so...', start=0x..., stop=0x..., size=0x..., flags=..., page_offset=...)
 
         The corefile also contains a :attr:`.stack` property, which gives
         us direct access to the stack contents.  On Linux, the very top of the stack
@@ -759,7 +759,7 @@ def vsyscall(self):
     @property
     def libc(self):
         """:class:`Mapping`: First mapping for ``libc.so``"""
-        expr = r'libc\b.*so$'
+        expr = r'^libc\b.*so(?:\.6)?$'
 
         for m in self.mappings:
             if not m.name:

pwnlib/gdb.py

@@ -1333,7 +1333,7 @@ def version(program='gdb'):
 
     Example:
 
-        >>> (7,0) <= gdb.version() <= (12,0)
+        >>> (7,0) <= gdb.version() <= (19,0)
         True
     """
     program = misc.which(program)

pwnlib/rop/rop.py

@@ -1224,6 +1224,9 @@ def __flat_at__(self, address):
 
     def __get_cachefile_name(self, files):
         """Given an ELF or list of ELF objects, return a cache file for the set of files"""
+        if context.cache_dir is None:
+            return None
+
         cachedir = os.path.join(context.cache_dir, 'rop-cache')
         if not os.path.exists(cachedir):
             os.mkdir(cachedir)
@@ -1240,21 +1243,26 @@ def __get_cachefile_name(self, files):
     @staticmethod
     def clear_cache():
         """Clears the ROP gadget cache"""
+        if context.cache_dir is None:
+            return
         cachedir = os.path.join(context.cache_dir, 'rop-cache')
         shutil.rmtree(cachedir)
 
     def __cache_load(self, elf):
         filename = self.__get_cachefile_name(elf)
-        if not os.path.exists(filename):
+        if filename is None or not os.path.exists(filename):
             return None
         gadgets = eval(open(filename).read())
         gadgets = {k - elf.load_addr + elf.address:v for k, v in gadgets.items()}
         log.info_once('Loaded %s cached gadgets for %r', len(gadgets), elf.file.name)
         return gadgets
 
     def __cache_save(self, elf, data):
+        filename = self.__get_cachefile_name(elf)
+        if filename is None:
+            return
         data = {k + elf.load_addr - elf.address:v for k, v in data.items()}
-        open(self.__get_cachefile_name(elf), 'w+').write(repr(data))
+        open(filename, 'w+').write(repr(data))
 
     def __load(self):
         """Load all ROP gadgets for the selected ELF files"""

pwnlib/shellcraft/internal.py

@@ -2,6 +2,7 @@
 from __future__ import division
 
 import os
+import sys
 
 from pwnlib.context import context
 
@@ -109,47 +110,17 @@ def get_context_from_dirpath(directory):
     return {'os': osys, 'arch': arch}
 
 def make_function(funcname, filename, directory):
+    import functools
     import inspect
     path       = os.path.join(directory, filename)
     template   = lookup_template(path)
 
-    args, varargs, keywords, defaults = inspect.getargspec(template.module.render_body)
-
-    defaults = defaults or []
-
-    if len(defaults) < len(args) and args[0] == 'context':
-        args.pop(0)
-
-    args_used = args[:]
-
-    for n, default in enumerate(defaults, len(args) - len(defaults)):
-        args[n] = '%s = %r' % (args[n], default)
-
-    if varargs:
-        args.append('*' + varargs)
-        args_used.append('*' + varargs)
-
-    if keywords not in ['pageargs', None]:
-        args.append('**' + keywords)
-        args_used.append('**' + keywords)
-
-    docstring = inspect.cleandoc(template.module.__doc__ or '')
-    args      = ', '.join(args)
-    args_used = ', '.join(args_used)
     local_ctx = get_context_from_dirpath(directory)
 
-    # This is a slight hack to get the right signature for the function
-    # It would be possible to simply create an (*args, **kwargs) wrapper,
-    # but what would not have the right signature.
-    # While we are at it, we insert the docstring too
-    T = r'''
-def wrap(template, render_global):
-    import pwnlib
-    def %(funcname)s(%(args)s):
-        %(docstring)r
+    def res(*args, **kwargs):
         with render_global.go_inside() as was_inside:
-            with pwnlib.context.context.local(**%(local_ctx)s):
-                lines = template.render(%(args_used)s).split('\n')
+            with context.local(**local_ctx):
+                lines = template.render(*args, **kwargs).split('\n')
         for i, line in enumerate(lines):
             def islabelchar(c):
                 return c.isalnum() or c == '.' or c == '_'
@@ -168,19 +139,16 @@ def islabelchar(c):
             return s
         else:
             return s + '\n'
-    return %(funcname)s
-''' % locals()
-
-    g = {}
-    exec(T, g, g)
-    wrap = g['wrap']
 
     # Setting _relpath is a slight hack only used to get better documentation
-    res = wrap(template, render_global)
     res._relpath = path
     res.__module__ = 'pwnlib.shellcraft.' + os.path.dirname(path).replace('/','.')
-
-    import sys, functools
+    res.__name__ = res.__qualname__ = funcname
+    res.__doc__ = inspect.cleandoc(template.module.__doc__ or '')
+    if hasattr(inspect, 'signature'):
+        sig = inspect.signature(template.module.render_body)
+        sig = sig.replace(parameters=list(sig.parameters.values())[1:-1])
+        res.__signature__ = sig
 
     @functools.wraps(res)
     def function(*a):

pwnlib/tubes/tube.py

@@ -196,14 +196,29 @@ def recvpred(self, pred, timeout = default):
         Returns:
             A bytes object containing bytes received from the socket,
             or ``''`` if a timeout occurred while waiting.
+
+        Examples:
+
+            >>> t = tube()
+            >>> t.recv_raw = lambda n: b'abbbaccc'
+            >>> pred = lambda p: p.count(b'a') == 2
+            >>> t.recvpred(pred)
+            b'abbba'
+            >>> pred = lambda p: p.count(b'd') > 0
+            >>> t.recvpred(pred, timeout=0.05)
+            b''
         """
 
         data = b''
 
         with self.countdown(timeout):
             while not pred(data):
+                if not self.countdown_active():
+                    self.unrecv(data)
+                    return b''
+
                 try:
-                    res = self.recv(1)
+                    res = self.recv(1, timeout=timeout)
                 except Exception:
                     self.unrecv(data)
                     return b''