Merge branch 'main' into pythongh-114271-remove-tstate_lock

.github/workflows/build.yml

@@ -484,6 +484,24 @@ jobs:
     - name: Tests
       run: xvfb-run make test
 
+  build_tsan:
+    name: 'Thread sanitizer'
+    needs: check_source
+    if: needs.check_source.outputs.run_tests == 'true'
+    uses: ./.github/workflows/reusable-tsan.yml
+    with:
+      config_hash: ${{ needs.check_source.outputs.config_hash }}
+      options: ./configure --config-cache --with-thread-sanitizer --with-pydebug
+
+  build_tsan_free_threading:
+    name: 'Thread sanitizer (free-threading)'
+    needs: check_source
+    if: needs.check_source.outputs.run_tests == 'true'
+    uses: ./.github/workflows/reusable-tsan.yml
+    with:
+      config_hash: ${{ needs.check_source.outputs.config_hash }}
+      options: ./configure --config-cache --disable-gil --with-thread-sanitizer --with-pydebug
+
   # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/
   cifuzz:
     name: CIFuzz
@@ -542,6 +560,8 @@ jobs:
     - build_windows_free_threading
     - test_hypothesis
     - build_asan
+    - build_tsan
+    - build_tsan_free_threading
     - cifuzz
 
     runs-on: ubuntu-latest
@@ -575,6 +595,8 @@ jobs:
             build_windows,
             build_windows_free_threading,
             build_asan,
+            build_tsan,
+            build_tsan_free_threading,
             '
             || ''
           }}

.github/workflows/reusable-tsan.yml

@@ -0,0 +1,51 @@
+on:
+  workflow_call:
+    inputs:
+      config_hash:
+        required: true
+        type: string
+      options:
+        required: true
+        type: string
+
+jobs:
+  build_tsan_reusable:
+    name: 'Thread sanitizer'
+    runs-on: ubuntu-22.04
+    timeout-minutes: 60
+    steps:
+    - uses: actions/checkout@v4
+    - name: Runner image version
+      run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV
+    - name: Restore config.cache
+      uses: actions/cache@v4
+      with:
+        path: config.cache
+        key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }}
+    - name: Install Dependencies
+      run: |
+        sudo ./.github/workflows/posix-deps-apt.sh
+        sudo apt install -y clang
+        # Reduce ASLR to avoid TSAN crashing
+        sudo sysctl -w vm.mmap_rnd_bits=28
+    - name: TSAN Option Setup
+      run: |
+        echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV
+        echo "CC=clang" >> $GITHUB_ENV
+        echo "CXX=clang++" >> $GITHUB_ENV
+    - name: Add ccache to PATH
+      run: |
+        echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
+    - name: Configure ccache action
+      uses: hendrikmuhs/[email protected]
+      with:
+        save: ${{ github.event_name == 'push' }}
+        max-size: "200M"
+    - name: Configure CPython
+      run: ${{ inputs.options }}
+    - name: Build CPython
+      run: make -j4
+    - name: Display build info
+      run: make pythoninfo
+    - name: Tests
+      run: ./python -m test --tsan -j4

Lib/statistics.py

@@ -963,7 +963,7 @@ def pdf(x):
             supported = sample[i : j]
             return sum(K((x - x_i) / h) for x_i in supported) / (n * h)
 
-    pdf.__doc__ = f'PDF estimate with {kernel=!r} and {h=!r}'
+    pdf.__doc__ = f'PDF estimate with {h=!r} and {kernel=!r}'
 
     return pdf
 

Lib/test/test_cmd_line.py

@@ -38,6 +38,7 @@ def verify_valid_flag(self, cmd_line):
         self.assertNotIn(b'Traceback', err)
         return out
 
+    @support.cpython_only
     def test_help(self):
         self.verify_valid_flag('-h')
         self.verify_valid_flag('-?')
@@ -48,14 +49,17 @@ def test_help(self):
         self.assertNotIn(b'-X dev', out)
         self.assertLess(len(lines), 50)
 
+    @support.cpython_only
     def test_help_env(self):
         out = self.verify_valid_flag('--help-env')
         self.assertIn(b'PYTHONHOME', out)
 
+    @support.cpython_only
     def test_help_xoptions(self):
         out = self.verify_valid_flag('--help-xoptions')
         self.assertIn(b'-X dev', out)
 
+    @support.cpython_only
     def test_help_all(self):
         out = self.verify_valid_flag('--help-all')
         lines = out.splitlines()
@@ -74,6 +78,7 @@ def test_optimize(self):
     def test_site_flag(self):
         self.verify_valid_flag('-S')
 
+    @support.cpython_only
     def test_version(self):
         version = ('Python %d.%d' % sys.version_info[:2]).encode("ascii")
         for switch in '-V', '--version', '-VV':
@@ -139,6 +144,7 @@ def run_python(*args):
         else:
             self.assertEqual(err, b'')
 
+    @support.cpython_only
     def test_xoption_frozen_modules(self):
         tests = {
             ('=on', 'FrozenImporter'),
@@ -153,6 +159,7 @@ def test_xoption_frozen_modules(self):
                 res = assert_python_ok(*cmd)
                 self.assertRegex(res.out.decode('utf-8'), expected)
 
+    @support.cpython_only
     def test_env_var_frozen_modules(self):
         tests = {
             ('on', 'FrozenImporter'),
@@ -579,6 +586,7 @@ def test_del___main__(self):
             print("del sys.modules['__main__']", file=script)
         assert_python_ok(filename)
 
+    @support.cpython_only
     def test_unknown_options(self):
         rc, out, err = assert_python_failure('-E', '-z')
         self.assertIn(b'Unknown option: -z', err)
@@ -691,6 +699,7 @@ def run_xdev(self, *args, check_exitcode=True, xdev=True):
             self.assertEqual(proc.returncode, 0, proc)
         return proc.stdout.rstrip()
 
+    @support.cpython_only
     def test_xdev(self):
         # sys.flags.dev_mode
         code = "import sys; print(sys.flags.dev_mode)"
@@ -914,6 +923,7 @@ def test_argv0_normalization(self):
         self.assertEqual(proc.returncode, 0, proc)
         self.assertEqual(proc.stdout.strip(), b'0')
 
+    @support.cpython_only
     def test_parsing_error(self):
         args = [sys.executable, '-I', '--unknown-option']
         proc = subprocess.run(args,

Lib/test/test_concurrent_futures/util.py

@@ -85,6 +85,8 @@ def get_context(self):
             self.skipTest("ProcessPoolExecutor unavailable on this system")
         if sys.platform == "win32":
             self.skipTest("require unix system")
+        if support.check_sanitizer(thread=True):
+            self.skipTest("TSAN doesn't support threads after fork")
         return super().get_context()
 
 
@@ -111,6 +113,8 @@ def get_context(self):
             self.skipTest("ProcessPoolExecutor unavailable on this system")
         if sys.platform == "win32":
             self.skipTest("require unix system")
+        if support.check_sanitizer(thread=True):
+            self.skipTest("TSAN doesn't support threads after fork")
         return super().get_context()
 
 

Lib/test/test_logging.py

@@ -80,6 +80,9 @@
 skip_if_asan_fork = unittest.skipIf(
     support.HAVE_ASAN_FORK_BUG,
     "libasan has a pthread_create() dead lock related to thread+fork")
+skip_if_tsan_fork = unittest.skipIf(
+    support.check_sanitizer(thread=True),
+    "TSAN doesn't support threads after fork")
 
 
 class BaseTest(unittest.TestCase):
@@ -731,6 +734,7 @@ def remove_loop(fname, tries):
     @support.requires_fork()
     @threading_helper.requires_working_threading()
     @skip_if_asan_fork
+    @skip_if_tsan_fork
     def test_post_fork_child_no_deadlock(self):
         """Ensure child logging locks are not held; bpo-6721 & bpo-36533."""
         class _OurHandler(logging.Handler):

Lib/test/test_threading.py

@@ -50,6 +50,11 @@ def skip_unless_reliable_fork(test):
     return test
 
 
+skip_if_tsan_fork = unittest.skipIf(
+    support.check_sanitizer(thread=True),
+    "TSAN doesn't support threads after fork")
+
+
 def requires_subinterpreters(meth):
     """Decorator to skip a test if subinterpreters are not supported."""
     return unittest.skipIf(interpreters is None,
@@ -634,6 +639,7 @@ def test_daemon_param(self):
         self.assertTrue(t.daemon)
 
     @skip_unless_reliable_fork
+    @skip_if_tsan_fork
     def test_dummy_thread_after_fork(self):
         # Issue #14308: a dummy thread in the active list doesn't mess up
         # the after-fork mechanism.
@@ -703,6 +709,7 @@ def f():
 
     @skip_unless_reliable_fork
     @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()")
+    @skip_if_tsan_fork
     def test_main_thread_after_fork(self):
         code = """if 1:
             import os, threading
@@ -1212,6 +1219,7 @@ def test_2_join_in_forked_process(self):
         self._run_and_join(script)
 
     @skip_unless_reliable_fork
+    @skip_if_tsan_fork
     def test_3_join_in_forked_from_thread(self):
         # Like the test above, but fork() was called from a worker thread
         # In the forked process, the main Thread object must be marked as stopped.