From b59a5a47d19199cfefa41b0f16a2c38487a584d2 Mon Sep 17 00:00:00 2001 From: Benjy Weinberger Date: Sat, 17 Dec 2022 15:07:01 -0800 Subject: [PATCH 1/3] Choose distinct debug adapter ports in different tests. Prevents a port collision error if the two test files happen to run concurrently. --- .../backend/python/goals/pytest_runner_integration_test.py | 3 +++ .../backend/python/goals/run_python_source_integration_test.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/python/pants/backend/python/goals/pytest_runner_integration_test.py b/src/python/pants/backend/python/goals/pytest_runner_integration_test.py index 9f649a00b86..20d5f90a3f0 100644 --- a/src/python/pants/backend/python/goals/pytest_runner_integration_test.py +++ b/src/python/pants/backend/python/goals/pytest_runner_integration_test.py @@ -108,6 +108,9 @@ def _configure_pytest_runner( args = [ "--backend-packages=pants.backend.python", f"--source-root-patterns={SOURCE_ROOT}", + # NB: Each test file that tests the debug adapter should pick a unique port + # so that different test files can run concurrently without port collisions. + "--debug-adapter-port=22335", *(extra_args or ()), ] rule_runner.set_options(args, env=env, env_inherit={"PATH", "PYENV_ROOT", "HOME"}) diff --git a/src/python/pants/backend/python/goals/run_python_source_integration_test.py b/src/python/pants/backend/python/goals/run_python_source_integration_test.py index 5bc3640bdc2..6fe256464fe 100644 --- a/src/python/pants/backend/python/goals/run_python_source_integration_test.py +++ b/src/python/pants/backend/python/goals/run_python_source_integration_test.py @@ -186,6 +186,9 @@ def my_file(): "--backend-packages=pants.backend.python", "--backend-packages=pants.backend.codegen.protobuf.python", "--source-root-patterns=['src_root1', 'src_root2']", + # NB: Each test file that tests the debug adapter should pick a unique port + # so that different test files can run concurrently without port collisions. + "--debug-adapter-port=22334", *( ( "--python-default-run-goal-use-sandbox" From 9157c6e5d60378781561d7adcfbe6edef36ae075 Mon Sep 17 00:00:00 2001 From: Benjy Weinberger Date: Sat, 17 Dec 2022 15:52:52 -0800 Subject: [PATCH 2/3] Use execution slot var --- pants.toml | 1 + .../python/goals/pytest_runner_integration_test.py | 5 ++--- .../goals/run_python_source_integration_test.py | 5 ++--- src/python/pants/core/subsystems/debug_adapter.py | 12 ++++++++++++ 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/pants.toml b/pants.toml index 47de44b4b87..2f25ef35eda 100644 --- a/pants.toml +++ b/pants.toml @@ -175,6 +175,7 @@ extra_requirements.add = [ "pygments", ] lockfile = "3rdparty/python/pytest.lock" +execution_slot_var = "TEST_EXECUTION_SLOT" [test] extra_env_vars = [ diff --git a/src/python/pants/backend/python/goals/pytest_runner_integration_test.py b/src/python/pants/backend/python/goals/pytest_runner_integration_test.py index 20d5f90a3f0..947fe299737 100644 --- a/src/python/pants/backend/python/goals/pytest_runner_integration_test.py +++ b/src/python/pants/backend/python/goals/pytest_runner_integration_test.py @@ -40,6 +40,7 @@ build_runtime_package_dependencies, get_filtered_environment, ) +from pants.core.subsystems.debug_adapter import DebugAdapterSubsystem from pants.core.util_rules import config_files, distdir from pants.core.util_rules.partitions import Partitions from pants.engine.addresses import Address @@ -108,9 +109,7 @@ def _configure_pytest_runner( args = [ "--backend-packages=pants.backend.python", f"--source-root-patterns={SOURCE_ROOT}", - # NB: Each test file that tests the debug adapter should pick a unique port - # so that different test files can run concurrently without port collisions. - "--debug-adapter-port=22335", + f"--debug-adapter-port={DebugAdapterSubsystem.port_for_testing()}", *(extra_args or ()), ] rule_runner.set_options(args, env=env, env_inherit={"PATH", "PYENV_ROOT", "HOME"}) diff --git a/src/python/pants/backend/python/goals/run_python_source_integration_test.py b/src/python/pants/backend/python/goals/run_python_source_integration_test.py index 6fe256464fe..2590216fc39 100644 --- a/src/python/pants/backend/python/goals/run_python_source_integration_test.py +++ b/src/python/pants/backend/python/goals/run_python_source_integration_test.py @@ -34,6 +34,7 @@ from pants.backend.python.util_rules import local_dists, pex_from_targets from pants.build_graph.address import Address from pants.core.goals.run import RunDebugAdapterRequest, RunRequest +from pants.core.subsystems.debug_adapter import DebugAdapterSubsystem from pants.engine.process import InteractiveProcess from pants.engine.rules import QueryRule from pants.engine.target import Target @@ -186,9 +187,7 @@ def my_file(): "--backend-packages=pants.backend.python", "--backend-packages=pants.backend.codegen.protobuf.python", "--source-root-patterns=['src_root1', 'src_root2']", - # NB: Each test file that tests the debug adapter should pick a unique port - # so that different test files can run concurrently without port collisions. - "--debug-adapter-port=22334", + f"--debug-adapter-port={DebugAdapterSubsystem.port_for_testing()}", *( ( "--python-default-run-goal-use-sandbox" diff --git a/src/python/pants/core/subsystems/debug_adapter.py b/src/python/pants/core/subsystems/debug_adapter.py index d43f6d5bfd8..456945e8c25 100644 --- a/src/python/pants/core/subsystems/debug_adapter.py +++ b/src/python/pants/core/subsystems/debug_adapter.py @@ -3,6 +3,8 @@ from __future__ import annotations +import os + from pants.option.option_types import IntOption, StrOption from pants.option.subsystem import Subsystem from pants.util.strutil import softwrap @@ -23,3 +25,13 @@ class DebugAdapterSubsystem(Subsystem): default=5678, help="The port to use when launching the server.", ) + + @staticmethod + def port_for_testing() -> int: + """Return a custom port we can use in Pants's own tests to avoid collisions. + + Assumes that the env var TEST_EXECUTION_SLOT has been set. If not, all tests will use the + same port, and collisions may occur. + """ + execution_slot = os.environ.get("TEST_EXECUTION_SLOT", "0") + return 22000 + int(execution_slot) From f94c506baad68a836285a12a0c2bfff59ed20d81 Mon Sep 17 00:00:00 2001 From: Benjy Weinberger Date: Tue, 20 Dec 2022 15:21:32 -0800 Subject: [PATCH 3/3] Move to util file --- .../goals/pytest_runner_integration_test.py | 4 ++-- .../goals/run_python_source_integration_test.py | 4 ++-- .../pants/core/subsystems/debug_adapter.py | 12 ------------ src/python/pants/testutil/debug_adapter_util.py | 16 ++++++++++++++++ 4 files changed, 20 insertions(+), 16 deletions(-) create mode 100644 src/python/pants/testutil/debug_adapter_util.py diff --git a/src/python/pants/backend/python/goals/pytest_runner_integration_test.py b/src/python/pants/backend/python/goals/pytest_runner_integration_test.py index 947fe299737..c5dd1747007 100644 --- a/src/python/pants/backend/python/goals/pytest_runner_integration_test.py +++ b/src/python/pants/backend/python/goals/pytest_runner_integration_test.py @@ -40,7 +40,6 @@ build_runtime_package_dependencies, get_filtered_environment, ) -from pants.core.subsystems.debug_adapter import DebugAdapterSubsystem from pants.core.util_rules import config_files, distdir from pants.core.util_rules.partitions import Partitions from pants.engine.addresses import Address @@ -49,6 +48,7 @@ from pants.engine.rules import Get, rule from pants.engine.target import Target from pants.engine.unions import UnionRule +from pants.testutil.debug_adapter_util import debugadapter_port_for_testing from pants.testutil.python_interpreter_selection import ( all_major_minor_python_versions, skip_unless_python27_and_python3_present, @@ -109,7 +109,7 @@ def _configure_pytest_runner( args = [ "--backend-packages=pants.backend.python", f"--source-root-patterns={SOURCE_ROOT}", - f"--debug-adapter-port={DebugAdapterSubsystem.port_for_testing()}", + f"--debug-adapter-port={debugadapter_port_for_testing()}", *(extra_args or ()), ] rule_runner.set_options(args, env=env, env_inherit={"PATH", "PYENV_ROOT", "HOME"}) diff --git a/src/python/pants/backend/python/goals/run_python_source_integration_test.py b/src/python/pants/backend/python/goals/run_python_source_integration_test.py index 2590216fc39..5de50859a10 100644 --- a/src/python/pants/backend/python/goals/run_python_source_integration_test.py +++ b/src/python/pants/backend/python/goals/run_python_source_integration_test.py @@ -34,10 +34,10 @@ from pants.backend.python.util_rules import local_dists, pex_from_targets from pants.build_graph.address import Address from pants.core.goals.run import RunDebugAdapterRequest, RunRequest -from pants.core.subsystems.debug_adapter import DebugAdapterSubsystem from pants.engine.process import InteractiveProcess from pants.engine.rules import QueryRule from pants.engine.target import Target +from pants.testutil.debug_adapter_util import debugadapter_port_for_testing from pants.testutil.pants_integration_test import run_pants from pants.testutil.rule_runner import RuleRunner, mock_console @@ -187,7 +187,7 @@ def my_file(): "--backend-packages=pants.backend.python", "--backend-packages=pants.backend.codegen.protobuf.python", "--source-root-patterns=['src_root1', 'src_root2']", - f"--debug-adapter-port={DebugAdapterSubsystem.port_for_testing()}", + f"--debug-adapter-port={debugadapter_port_for_testing()}", *( ( "--python-default-run-goal-use-sandbox" diff --git a/src/python/pants/core/subsystems/debug_adapter.py b/src/python/pants/core/subsystems/debug_adapter.py index 456945e8c25..d43f6d5bfd8 100644 --- a/src/python/pants/core/subsystems/debug_adapter.py +++ b/src/python/pants/core/subsystems/debug_adapter.py @@ -3,8 +3,6 @@ from __future__ import annotations -import os - from pants.option.option_types import IntOption, StrOption from pants.option.subsystem import Subsystem from pants.util.strutil import softwrap @@ -25,13 +23,3 @@ class DebugAdapterSubsystem(Subsystem): default=5678, help="The port to use when launching the server.", ) - - @staticmethod - def port_for_testing() -> int: - """Return a custom port we can use in Pants's own tests to avoid collisions. - - Assumes that the env var TEST_EXECUTION_SLOT has been set. If not, all tests will use the - same port, and collisions may occur. - """ - execution_slot = os.environ.get("TEST_EXECUTION_SLOT", "0") - return 22000 + int(execution_slot) diff --git a/src/python/pants/testutil/debug_adapter_util.py b/src/python/pants/testutil/debug_adapter_util.py new file mode 100644 index 00000000000..2abd9c9883b --- /dev/null +++ b/src/python/pants/testutil/debug_adapter_util.py @@ -0,0 +1,16 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +import os + + +def debugadapter_port_for_testing() -> int: + """Return a unique-per-concurrent-process debug adapter port. + + Use this in Pants's (and plugins') own tests to avoid collisions. + + Assumes that the env var TEST_EXECUTION_SLOT has been set. If not, all tests + will use the same port, and collisions may occur. + """ + execution_slot = os.environ.get("TEST_EXECUTION_SLOT", "0") + return 22000 + int(execution_slot)