From b9e8b43f83e2154249126800d58bc40aa7074d29 Mon Sep 17 00:00:00 2001 From: Fridolin Pokorny Date: Mon, 8 Feb 2021 13:56:52 +0100 Subject: [PATCH] Implement a pipeline unit for selective pre-release filtering --- tests/sieves/test_experimental_prereleases.py | 191 ++++++++++++++++++ thoth/adviser/sieves/__init__.py | 2 + .../sieves/experimental_prereleases.py | 78 +++++++ 3 files changed, 271 insertions(+) create mode 100644 tests/sieves/test_experimental_prereleases.py create mode 100644 thoth/adviser/sieves/experimental_prereleases.py diff --git a/tests/sieves/test_experimental_prereleases.py b/tests/sieves/test_experimental_prereleases.py new file mode 100644 index 000000000..b95cc4ab0 --- /dev/null +++ b/tests/sieves/test_experimental_prereleases.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +# thoth-adviser +# Copyright(C) 2021 Fridolin Pokorny +# +# This program is free software: you can redistribute it and / or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Test removing pre-releases based on Pipfile configuration in Thoth section.""" + +import itertools + +import flexmock +import pytest + +from thoth.adviser.sieves import SelectiveCutPreReleasesSieve +from thoth.adviser.enums import RecommendationType +from thoth.adviser.pipeline_builder import PipelineBuilderContext +from thoth.python import Source +from thoth.python import PackageVersion +from thoth.python import Project + +from ..base import AdviserUnitTestCase + + +class TestSelctiveCutPreReleasesSieve(AdviserUnitTestCase): + """Test removing pre-releases based on Pipfile configuration in Thoth section.""" + + UNIT_TESTED = SelectiveCutPreReleasesSieve + + _CASE_GLOBAL_DISALLOWED_PIPFILE = """ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +tensorflow = "*" + +[pipenv] +allow_prereleases = false + +[thoth.allow_prereleases] +tensorflow = true +flask = false +""" + + _CASE_GLOBAL_DISALLOWED_EMPTY_PIPFILE = """ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +tensorflow = "*" + +[pipenv] +allow_prereleases = false +""" + + _CASE_GLOBAL_ALLOWED_PIPFILE = """ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +tensorflow = "*" + +[pipenv] +allow_prereleases = true + +[thoth.allow_prereleases] +tensorflow = true +flask = false +""" + + @pytest.mark.parametrize( + "recommendation_type", + [ + RecommendationType.STABLE, + RecommendationType.PERFORMANCE, + RecommendationType.SECURITY, + RecommendationType.LATEST, + RecommendationType.TESTING, + ], + ) + def test_include( + self, + builder_context: PipelineBuilderContext, + recommendation_type: RecommendationType, + ) -> None: + """Test including this pipeline unit.""" + builder_context.recommendation_type = recommendation_type + builder_context.project = Project.from_strings(self._CASE_GLOBAL_DISALLOWED_PIPFILE) + + assert builder_context.is_adviser_pipeline() + assert self.UNIT_TESTED.should_include(builder_context) == { + "package_name": None, + "allow_prereleases": { + "tensorflow": True, + "flask": False, + }, + } + + def test_verify_multiple_should_include(self, builder_context: PipelineBuilderContext) -> None: + """Verify multiple should_include calls do not loop endlessly.""" + builder_context.recommendation_type = RecommendationType.LATEST + builder_context.project = Project.from_strings(self._CASE_GLOBAL_DISALLOWED_PIPFILE) + self.verify_multiple_should_include(builder_context) + + @pytest.mark.skip(reason="The pipeline unit configuration is specific to Pipfile configuration") + def test_default_configuration(self, builder_context: PipelineBuilderContext) -> None: + """Test the default configuration.""" + + @pytest.mark.parametrize( + "recommendation_type,case", + itertools.product( + [ + RecommendationType.STABLE, + RecommendationType.PERFORMANCE, + RecommendationType.SECURITY, + RecommendationType.LATEST, + RecommendationType.TESTING, + ], + [_CASE_GLOBAL_ALLOWED_PIPFILE, _CASE_GLOBAL_DISALLOWED_EMPTY_PIPFILE], + ), + ) + def test_not_include( + self, + builder_context: PipelineBuilderContext, + recommendation_type: RecommendationType, + case: str, + ) -> None: + """Test not including this pipeline unit.""" + builder_context.recommendation_type = recommendation_type + builder_context.project = Project.from_strings(self._CASE_GLOBAL_ALLOWED_PIPFILE) + + assert builder_context.is_adviser_pipeline() + assert self.UNIT_TESTED.should_include(builder_context) is None + + @pytest.mark.parametrize("package_name,package_version", [("tensorflow", "2.4.0rc1"), ("flask", "1.0")]) + def test_remove_pre_releases_disallowed_noop(self, package_name: str, package_version: str) -> None: + """Test NOT removing dependencies based on pre-release configuration.""" + pv = PackageVersion( + name=package_name, + version=f"=={package_version}", + index=Source("https://pypi.org/simple"), + develop=False, + ) + + project = Project.from_strings(self._CASE_GLOBAL_DISALLOWED_PIPFILE) + context = flexmock(project=project) + sieve = self.UNIT_TESTED() + sieve.update_configuration({"package_name": None, "allow_prereleases": project.pipfile.thoth.allow_prereleases}) + + with self.UNIT_TESTED.assigned_context(context): + assert list(sieve.run(p for p in [pv])) == [pv] + + @pytest.mark.parametrize( + "package_name,package_version", + [ + ("flask", "1.0dev0"), # Disabled explicitly. + ("numpy", "1.0dev0"), # Disabled implicitly. + ], + ) + def test_pre_releases_disallowed_removal(self, package_name: str, package_version: str) -> None: + """Test no removals if pre-releases are allowed.""" + pv = PackageVersion( + name=package_name, + version=f"=={package_version}", + index=Source("https://pypi.org/simple"), + develop=False, + ) + + project = Project.from_strings(self._CASE_GLOBAL_DISALLOWED_PIPFILE) + context = flexmock(project=project) + sieve = self.UNIT_TESTED() + sieve.update_configuration({"package_name": None, "allow_prereleases": project.pipfile.thoth.allow_prereleases}) + + with self.UNIT_TESTED.assigned_context(context): + assert list(sieve.run(p for p in [pv])) == [] diff --git a/thoth/adviser/sieves/__init__.py b/thoth/adviser/sieves/__init__.py index 737c674a5..648d9cd08 100644 --- a/thoth/adviser/sieves/__init__.py +++ b/thoth/adviser/sieves/__init__.py @@ -27,6 +27,7 @@ from .locked import CutLockedSieve from .pandas import PandasPy36Sieve from .prereleases import CutPreReleasesSieve +from .experimental_prereleases import SelectiveCutPreReleasesSieve from .setuptools import Py36SetuptoolsSieve from .solved import SolvedSieve from .tensorflow import TensorFlow240AVX2IllegalInstructionSieve @@ -42,6 +43,7 @@ # can be mentioned here. __all__ = [ "CutPreReleasesSieve", + "SelectiveCutPreReleasesSieve", "CutLockedSieve", "PackageIndexSieve", "SolvedSieve", diff --git a/thoth/adviser/sieves/experimental_prereleases.py b/thoth/adviser/sieves/experimental_prereleases.py new file mode 100644 index 000000000..800d81763 --- /dev/null +++ b/thoth/adviser/sieves/experimental_prereleases.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +# thoth-adviser +# Copyright(C) 2021 Fridolin Pokorny +# +# This program is free software: you can redistribute it and / or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""A sieve to filter out pre-releases, selectively.""" + +import logging +from typing import Optional +from typing import Dict +from typing import Any +from typing import Generator +from typing import TYPE_CHECKING + +import attr +from thoth.python import PackageVersion +from voluptuous import Schema +from voluptuous import Required + +from ..sieve import Sieve + +if TYPE_CHECKING: + from ..pipeline_builder import PipelineBuilderContext + +_LOGGER = logging.getLogger(__name__) + + +@attr.s(slots=True) +class SelectiveCutPreReleasesSieve(Sieve): + """Enable or disable specific pre-releases for the given set of packages..""" + + CONFIGURATION_DEFAULT = {"package_name": None, "allow_prereleases": None} + CONFIGURATION_SCHEMA: Schema = Schema( + {Required("package_name"): None, Required("allow_prereleases"): Schema({str: bool})} + ) + + @classmethod + def should_include(cls, builder_context: "PipelineBuilderContext") -> Optional[Dict[str, Any]]: + """Enable or disable specific pre-releases for the given set of packages..""" + if builder_context.project.prereleases_allowed: + if builder_context.project.prereleases_allowed: + msg = "Ignoring selective pre-releases in [thoth] section as global allow_prereleases flag is set" + _LOGGER.warning(msg) + + return None + + if builder_context.is_included(cls) or not builder_context.project.pipfile.thoth.allow_prereleases: + return None + + return { + "package_name": None, + "allow_prereleases": builder_context.project.pipfile.thoth.allow_prereleases, + } + + def run(self, package_versions: Generator[PackageVersion, None, None]) -> Generator[PackageVersion, None, None]: + """Cut-off pre-releases if project does not explicitly allows them.""" + for package_version in package_versions: + if not self.configuration["allow_prereleases"].get(package_version.name, False): + if package_version.semantic_version.is_prerelease: + _LOGGER.debug( + "Removing package %s - pre-releases are disabled", + package_version.to_tuple(), + ) + continue + + yield package_version