From b9a14fef53c0db1542989dae2e05f0234532e4ac Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Tue, 1 Dec 2020 17:43:13 +0000 Subject: [PATCH 1/5] test(python): use constraints files to check dependency lower bounds --- .../gcp/templates/python_library/noxfile.py.j2 | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/synthtool/gcp/templates/python_library/noxfile.py.j2 b/synthtool/gcp/templates/python_library/noxfile.py.j2 index b54ce9815..97ce742d8 100644 --- a/synthtool/gcp/templates/python_library/noxfile.py.j2 +++ b/synthtool/gcp/templates/python_library/noxfile.py.j2 @@ -18,6 +18,7 @@ from __future__ import absolute_import import os +import pathlib import shutil import nox @@ -30,6 +31,8 @@ DEFAULT_PYTHON_VERSION="{{ default_python_version }}" SYSTEM_TEST_PYTHON_VERSIONS=[{% for v in system_test_python_versions %}"{{v}}"{% if not loop.last %},{% endif %}{% endfor %}] UNIT_TEST_PYTHON_VERSIONS=[{% for v in unit_test_python_versions %}"{{v}}"{% if not loop.last %},{% endif %}{% endfor %}] +CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() + @nox.session(python=DEFAULT_PYTHON_VERSION) def lint(session): """Run linters. @@ -72,12 +75,14 @@ def lint_setup_py(session): def default(session): # Install all test dependencies, then install this package in-place. - + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) {%- if microgenerator %} session.install("asyncmock", "pytest-asyncio") {% endif %} session.install("mock", "pytest", "pytest-cov", {% for d in unit_test_external_dependencies %}"{{d}}"{% if not loop.last %},{% endif %}{% endfor %}) - session.install("-e", ".") + session.install("-e", ".", "-c", constraints_path) {% for dependency in unit_test_local_dependencies %}session.install("-e", "{{dependency}}"){% endfor %} {% for dependency in unit_test_dependencies %}session.install("-e", "{{dependency}}"){% endfor %} @@ -105,6 +110,9 @@ def unit(session): @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) def system(session): """Run the system test suite.""" + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) system_test_path = os.path.join("tests", "system.py") system_test_folder_path = os.path.join("tests", "system") @@ -131,7 +139,7 @@ def system(session): {%- if system_test_local_dependencies %} session.install("-e", {% for d in system_test_local_dependencies %}"{{d}}"{% if not loop.last %},{% endif %}{% endfor %}) {%- endif %} - session.install("-e", ".") + session.install("-e", ".", "-c", constraints_path) # Run py.test against the system tests. From 9c6f155b489923bf7d4414ef9cc6731762be6208 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Thu, 3 Dec 2020 22:01:53 +0000 Subject: [PATCH 2/5] fix: don't fail if no constraints file is found --- .../gcp/templates/python_library/noxfile.py.j2 | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/synthtool/gcp/templates/python_library/noxfile.py.j2 b/synthtool/gcp/templates/python_library/noxfile.py.j2 index 97ce742d8..7f9f7e187 100644 --- a/synthtool/gcp/templates/python_library/noxfile.py.j2 +++ b/synthtool/gcp/templates/python_library/noxfile.py.j2 @@ -75,14 +75,18 @@ def lint_setup_py(session): def default(session): # Install all test dependencies, then install this package in-place. - constraints_path = str( + + constraints_path = ( CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" ) {%- if microgenerator %} session.install("asyncmock", "pytest-asyncio") {% endif %} session.install("mock", "pytest", "pytest-cov", {% for d in unit_test_external_dependencies %}"{{d}}"{% if not loop.last %},{% endif %}{% endfor %}) - session.install("-e", ".", "-c", constraints_path) + if constraints_path.exists(): + session.install("-e", ".", "-c", str(constraints_path)) + else: + session.install("-e", ".") {% for dependency in unit_test_local_dependencies %}session.install("-e", "{{dependency}}"){% endfor %} {% for dependency in unit_test_dependencies %}session.install("-e", "{{dependency}}"){% endfor %} @@ -110,7 +114,7 @@ def unit(session): @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) def system(session): """Run the system test suite.""" - constraints_path = str( + constraints_path = ( CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" ) system_test_path = os.path.join("tests", "system.py") @@ -139,7 +143,10 @@ def system(session): {%- if system_test_local_dependencies %} session.install("-e", {% for d in system_test_local_dependencies %}"{{d}}"{% if not loop.last %},{% endif %}{% endfor %}) {%- endif %} - session.install("-e", ".", "-c", constraints_path) + if constraints_path.exists(): + session.install("-e", ".", "-c", str(constraints_path)) + else: + session.install("-e", ".") # Run py.test against the system tests. From dfda1a015742857ac8a88dbdb34b262027590a42 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Thu, 3 Dec 2020 22:32:12 +0000 Subject: [PATCH 3/5] fix: fail on missing file; add constraints to all dep installs --- .../gcp/templates/python_library/noxfile.py.j2 | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/synthtool/gcp/templates/python_library/noxfile.py.j2 b/synthtool/gcp/templates/python_library/noxfile.py.j2 index d9068f93d..56d0b62ae 100644 --- a/synthtool/gcp/templates/python_library/noxfile.py.j2 +++ b/synthtool/gcp/templates/python_library/noxfile.py.j2 @@ -80,13 +80,10 @@ def default(session): CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" ) {%- if microgenerator %} - session.install("asyncmock", "pytest-asyncio") + session.install("asyncmock", "pytest-asyncio", "-c", str(constraints_path)) {% endif %} - session.install("mock", "pytest", "pytest-cov", {% for d in unit_test_external_dependencies %}"{{d}}"{% if not loop.last %},{% endif %}{% endfor %}) - if constraints_path.exists(): - session.install("-e", ".", "-c", str(constraints_path)) - else: - session.install("-e", ".") + session.install("mock", "pytest", "pytest-cov"{% for d in unit_test_external_dependencies %}, "{{d}}"{% endfor %}, "-c", str(constraints_path)) + session.install("-e", ".", "-c", str(constraints_path)) {% for dependency in unit_test_local_dependencies %}session.install("-e", "{{dependency}}"){% endfor %} {% for dependency in unit_test_dependencies %}session.install("-e", "{{dependency}}"){% endfor %} @@ -137,15 +134,12 @@ def system(session): # Install all test dependencies, then install this package into the # virtualenv's dist-packages. - session.install("mock", "pytest", "google-cloud-testutils", {% for d in system_test_external_dependencies %}"{{d}}"{% if not loop.last %},{% endif %}{% endfor %}) + session.install("mock", "pytest", "google-cloud-testutils"{% for d in system_test_external_dependencies %}, "{{d}}"{% endfor %}, "-c", str(constraints_path)) {%- if system_test_local_dependencies %} session.install("-e", {% for d in system_test_local_dependencies %}"{{d}}"{% if not loop.last %},{% endif %}{% endfor %}) {%- endif %} - if constraints_path.exists(): - session.install("-e", ".", "-c", str(constraints_path)) - else: - session.install("-e", ".") + session.install("-e", ".", "-c", str(constraints_path)) # Run py.test against the system tests. From 2a290475c184e929c423d5ba1f0f0a25ce998e40 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Thu, 3 Dec 2020 22:34:45 +0000 Subject: [PATCH 4/5] chore: use path as string --- .../gcp/templates/python_library/noxfile.py.j2 | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/synthtool/gcp/templates/python_library/noxfile.py.j2 b/synthtool/gcp/templates/python_library/noxfile.py.j2 index 56d0b62ae..d0d669d47 100644 --- a/synthtool/gcp/templates/python_library/noxfile.py.j2 +++ b/synthtool/gcp/templates/python_library/noxfile.py.j2 @@ -76,14 +76,14 @@ def lint_setup_py(session): def default(session): # Install all test dependencies, then install this package in-place. - constraints_path = ( + constraints_path = str( CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" ) {%- if microgenerator %} - session.install("asyncmock", "pytest-asyncio", "-c", str(constraints_path)) + session.install("asyncmock", "pytest-asyncio", "-c", constraints_path) {% endif %} - session.install("mock", "pytest", "pytest-cov"{% for d in unit_test_external_dependencies %}, "{{d}}"{% endfor %}, "-c", str(constraints_path)) - session.install("-e", ".", "-c", str(constraints_path)) + session.install("mock", "pytest", "pytest-cov"{% for d in unit_test_external_dependencies %}, "{{d}}"{% endfor %}, "-c", constraints_path) + session.install("-e", ".", "-c", constraints_path) {% for dependency in unit_test_local_dependencies %}session.install("-e", "{{dependency}}"){% endfor %} {% for dependency in unit_test_dependencies %}session.install("-e", "{{dependency}}"){% endfor %} @@ -110,7 +110,7 @@ def unit(session): @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) def system(session): """Run the system test suite.""" - constraints_path = ( + constraints_path = str( CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" ) system_test_path = os.path.join("tests", "system.py") @@ -134,12 +134,12 @@ def system(session): # Install all test dependencies, then install this package into the # virtualenv's dist-packages. - session.install("mock", "pytest", "google-cloud-testutils"{% for d in system_test_external_dependencies %}, "{{d}}"{% endfor %}, "-c", str(constraints_path)) + session.install("mock", "pytest", "google-cloud-testutils"{% for d in system_test_external_dependencies %}, "{{d}}"{% endfor %}, "-c", constraints_path) {%- if system_test_local_dependencies %} session.install("-e", {% for d in system_test_local_dependencies %}"{{d}}"{% if not loop.last %},{% endif %}{% endfor %}) {%- endif %} - session.install("-e", ".", "-c", str(constraints_path)) + session.install("-e", ".", "-c", constraints_path) # Run py.test against the system tests. From 897f08f88101a25b7c4f520c7fbcdf399b784d2c Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 23 Mar 2021 11:43:31 -0500 Subject: [PATCH 5/5] fix tests --- .../templates/python_library/noxfile.py.j2 | 2 +- tests/test_python_library.py | 22 +++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/synthtool/gcp/templates/python_library/noxfile.py.j2 b/synthtool/gcp/templates/python_library/noxfile.py.j2 index faeb6fcf6..149fd5a84 100644 --- a/synthtool/gcp/templates/python_library/noxfile.py.j2 +++ b/synthtool/gcp/templates/python_library/noxfile.py.j2 @@ -97,7 +97,7 @@ def default(session): {%- if microgenerator %} session.install("asyncmock", "pytest-asyncio", "-c", constraints_path) {% endif %} - session.install("mock", "pytest", "pytest-cov", {% for d in unit_test_external_dependencies %}"{{d}}"{% if not loop.last %},{% endif %}{% endfor %}, "-c", constraints_path) + session.install("mock", "pytest", "pytest-cov", {% for d in unit_test_external_dependencies %}"{{d}}",{% endfor %} "-c", constraints_path) {% for dependency in unit_test_local_dependencies %}session.install("-e", "{{dependency}}", "-c", constraints_path) {% endfor %} {% for dependency in unit_test_dependencies %}session.install("-e", "{{dependency}}", "-c", constraints_path){% endfor %} diff --git a/tests/test_python_library.py b/tests/test_python_library.py index a7a22368c..fdcef75a8 100644 --- a/tests/test_python_library.py +++ b/tests/test_python_library.py @@ -27,35 +27,35 @@ @pytest.mark.parametrize( ["template_kwargs", "expected_text"], [ - ({}, ["import nox", 'session.install("-e", ".")']), + ({}, ["import nox", 'session.install("-e", ".", "-c", constraints_path)']), ( {"unit_test_local_dependencies": ["../testutils", "../unitutils"]}, [ - 'session.install("-e", "../testutils")', - 'session.install("-e", "../unitutils")', + 'session.install("-e", "../testutils", "-c", constraints_path)', + 'session.install("-e", "../unitutils", "-c", constraints_path)', ], ), ( {"system_test_local_dependencies": ["../testutils", "../sysutils"]}, [ - 'session.install("-e", "../testutils")', - 'session.install("-e", "../sysutils")', + 'session.install("-e", "../testutils", "-c", constraints_path)', + 'session.install("-e", "../sysutils", "-c", constraints_path)', ], ), ( {"unit_test_extras": ["abc", "def"]}, - ['session.install("-e", ".[abc,def]")'], + ['session.install("-e", ".[abc,def]", "-c", constraints_path)'], ), ( {"system_test_extras": ["abc", "def"]}, - ['session.install("-e", ".[abc,def]")'], + ['session.install("-e", ".[abc,def]", "-c", constraints_path)'], ), ( {"unit_test_extras_by_python": {"3.8": ["abc", "def"]}}, [ 'if session.python == "3.8":\n extras = "[abc,def]"', 'else:\n extras = ""', - 'session.install("-e", f".{extras}")', + 'session.install("-e", f".{extras}", "-c", constraints_path)', ], ), ( @@ -63,7 +63,7 @@ [ 'if session.python == "3.8":\n extras = "[abc,def]"', 'else:\n extras = ""', - 'session.install("-e", f".{extras}")', + 'session.install("-e", f".{extras}", "-c", constraints_path)', ], ), ( @@ -74,7 +74,7 @@ [ 'if session.python == "3.8":\n extras = "[abc,def]"', 'else:\n extras = "[tuv,wxyz]"', - 'session.install("-e", f".{extras}")', + 'session.install("-e", f".{extras}", "-c", constraints_path)', ], ), ( @@ -85,7 +85,7 @@ [ 'if session.python == "3.8":\n extras = "[abc,def]"', 'else:\n extras = "[tuv,wxyz]"', - 'session.install("-e", f".{extras}")', + 'session.install("-e", f".{extras}", "-c", constraints_path)', ], ), ],