From e144575a62d98e9b07ff25769a324c770e5d65b1 Mon Sep 17 00:00:00 2001 From: Fridolin Pokorny Date: Fri, 29 Jan 2021 11:11:56 +0100 Subject: [PATCH] Filter out packages based on ABI symbols present in S2I Thoth --- tests/sieves/test_thoth_s2i_abi_compat.py | 130 +++++++++++++++++++ thoth/adviser/sieves/thoth_s2i_abi_compat.py | 125 ++++++++++++++++++ 2 files changed, 255 insertions(+) create mode 100644 tests/sieves/test_thoth_s2i_abi_compat.py create mode 100644 thoth/adviser/sieves/thoth_s2i_abi_compat.py diff --git a/tests/sieves/test_thoth_s2i_abi_compat.py b/tests/sieves/test_thoth_s2i_abi_compat.py new file mode 100644 index 000000000..21572c8b2 --- /dev/null +++ b/tests/sieves/test_thoth_s2i_abi_compat.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +# thoth-adviser +# Copyright(C) 2020 Kevin Postlethwait +# +# This program is free software: you can redistribute it and / or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Test filtering out Python Packages based on required and available ABI symbols.""" + +from typing import Optional + +import flexmock +import pytest + +from thoth.adviser.context import Context +from thoth.adviser.enums import RecommendationType +from thoth.adviser.pipeline_builder import PipelineBuilderContext +from thoth.adviser.sieves import ThothS2IAbiCompatibilitySieve +from thoth.python import PackageVersion +from thoth.python import Source +from thoth.storages import GraphDatabase + +from ..base import AdviserUnitTestCase + +_SYSTEM_SYMBOLS = ["GLIBC_2.0", "GLIBC_2.1", "GLIBC_2.2", "GLIBC_2.3", "GLIBC_2.4", "GLIBC_2.5", "GCC_3.4", "X_2.21"] +_REQUIRED_SYMBOLS_A = ["GLIBC_2.9"] +_REQUIRED_SYMBOLS_B = ["GLIBC_2.4"] + + +class TestThothS2IAbiCompatibilitySieve(AdviserUnitTestCase): + """Test filtering out packages based on symbols required.""" + + UNIT_TESTED = ThothS2IAbiCompatibilitySieve + + def test_verify_multiple_should_include(self, builder_context: PipelineBuilderContext) -> None: + """Verify multiple should_include calls do not loop endlessly.""" + builder_context.recommendation_type = RecommendationType.LATEST + builder_context.project.runtime_environment.base_image = "quay.io/thoth-station/s2i-thoth-ubi8-py38:v0.23.0" + self.verify_multiple_should_include(builder_context) + + @pytest.mark.parametrize("base_image", [ + None, + "fedora:32", + ]) + def test_no_should_include(self, base_image: Optional[str], builder_context: PipelineBuilderContext) -> None: + """Test not including this pipeline unit.""" + builder_context.project.runtime_environment.base_image = base_image + assert self.UNIT_TESTED.should_include(builder_context) is None + + def test_should_include(self, builder_context: PipelineBuilderContext) -> None: + """Test including this pipeline unit.""" + builder_context.project.runtime_environment.base_image = "quay.io/thoth-station/s2i-thoth-ubi8-py38:v0.23.0" + assert self.UNIT_TESTED.should_include(builder_context) == {} + + def test_no_thoth_s2i_version(self, context: Context) -> None: + """Test no Thoth S2I version present.""" + context.project.runtime_environment.base_image = "quay.io/thoth-station/s2i-thoth-ubi8-py38" # No version. + + GraphDatabase.should_receive("get_thoth_s2i_analyzed_image_symbols_all").times(0) + + unit = self.UNIT_TESTED() + with self.UNIT_TESTED.assigned_context(context): + unit.pre_run() + + assert not unit.image_symbols + + def test_abi_compat_symbols_present(self, context: Context) -> None: + """Test if required symbols are correctly identified.""" + source = Source("https://pypi.org/simple") + package_version = PackageVersion(name="tensorflow", version="==1.9.0", index=source, develop=False) + flexmock(GraphDatabase) + GraphDatabase.should_receive("get_thoth_s2i_analyzed_image_symbols_all").and_return(_SYSTEM_SYMBOLS).once() + GraphDatabase.should_receive("get_python_package_required_symbols").and_return(_REQUIRED_SYMBOLS_B).once() + + context.project.runtime_environment = flexmock( + operating_system=flexmock(name="rhel", version="8.0"), + cuda_version="4.6", + python_version="3.6", + ) + + with self.UNIT_TESTED.assigned_context(context): + sieve = self.UNIT_TESTED() + sieve.pre_run() + assert list(sieve.run((p for p in [package_version]))) == [package_version] + + def test_abi_compat_symbols_not_present(self, context: Context) -> None: + """Test if required symbols being missing is correctly identified.""" + source = Source("https://pypi.org/simple") + package_version = PackageVersion(name="tensorflow", version="==1.9.0", index=source, develop=False) + flexmock(GraphDatabase) + GraphDatabase.should_receive("get_thoth_s2i_analyzed_image_symbols_all").and_return(_SYSTEM_SYMBOLS).once() + GraphDatabase.should_receive("get_python_package_required_symbols").and_return(_REQUIRED_SYMBOLS_A).once() + + context.project.runtime_environment = flexmock( + operating_system=flexmock(name="rhel", version="8.0"), + cuda_version="4.6", + python_version="3.6", + ) + + with self.UNIT_TESTED.assigned_context(context): + sieve = self.UNIT_TESTED() + sieve.pre_run() + assert list(sieve.run((p for p in [package_version]))) == [] + + def test_super_pre_run(self, context: Context) -> None: + """Make sure the pre-run method of the base is called.""" + context.graph.should_receive("get_analyzed_image_symbols_all").with_args( + os_name=context.project.runtime_environment.operating_system.name, + os_version=context.project.runtime_environment.operating_system.version, + cuda_version=context.project.runtime_environment.cuda_version, + python_version=context.project.runtime_environment.python_version, + ).and_return(set()).once() + + unit = self.UNIT_TESTED() + assert unit.unit_run is False + + with unit.assigned_context(context): + unit.pre_run() + + assert unit.unit_run is False diff --git a/thoth/adviser/sieves/thoth_s2i_abi_compat.py b/thoth/adviser/sieves/thoth_s2i_abi_compat.py new file mode 100644 index 000000000..39575b757 --- /dev/null +++ b/thoth/adviser/sieves/thoth_s2i_abi_compat.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +# thoth-adviser +# Copyright(C) 2019-2021 Kevin Postlehtwait, Fridolin Pokorny +# +# This program is free software: you can redistribute it and / or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Filter out stacks which have require non-existent ABI symbols in Thoth's s2i base image.""" + +import logging +from typing import Any, Dict, Optional, Generator, Set, TYPE_CHECKING, Tuple + +import attr +from thoth.common import get_justification_link as jl +from thoth.python import PackageVersion + +from ..sieve import Sieve + +if TYPE_CHECKING: + from ..pipeline_builder import PipelineBuilderContext + +_LOGGER = logging.getLogger(__name__) + + +@attr.s(slots=True) +class ThothS2IAbiCompatibilitySieve(Sieve): + """Remove packages if the Thoth's s2i image being used doesn't have necessary ABI.""" + + _THOTH_S2I_PREFIX = "quay.io/thoth-station/" + CONFIGURATION_DEFAULT = {"package_name": None} + image_symbols = attr.ib(type=Set[str], factory=set, init=False) + _messages_logged = attr.ib(type=Set[Tuple[str, str, str]], factory=set, init=False) + + _LINK = jl("abi_missing") + _LINK_BAD_IMAGE = jl("bad_base_image") + + @classmethod + def should_include(cls, builder_context: "PipelineBuilderContext") -> Optional[Dict[str, Any]]: + """Register if the base image provided is Thoth's s2i.""" + if builder_context.is_included(cls): + return None + + base_image = builder_context.project.runtime_environment.base_image + if base_image and base_image.startswith(cls._THOTH_S2I_PREFIX): + return {} + + return None + + def pre_run(self) -> None: + """Initialize image_symbols.""" + base_image = self.context.project.runtime_environment.base_image + parts = base_image.split(":", maxsplit=1) + if len(parts) != 2: + error_msg = f"Cannot determine Thoth s2i version information from {base_image}, "\ + "recommendations specific for ABI used will not be taken into account" + _LOGGER.warning("%s - see %s", error_msg, self._LINK_BAD_IMAGE) + self.context.stack_info.append({ + "type": "WARNING", + "message": error_msg, + "link": self._LINK_BAD_IMAGE, + }) + + self.image_symbols = set() + return + + thoth_s2i_image_name, thoth_s2i_image_version = parts + if thoth_s2i_image_version.startswith("v"): + # Not nice as we always prefix with "v" but do not store it with "v" in the database + # (based on env var exported and detected in Thoth's s2i). + thoth_s2i_image_version = thoth_s2i_image_version[1:] + + self.image_symbols = set( + self.context.graph.get_thoth_s2i_analyzed_image_symbols_all( + thoth_s2i_image_name=thoth_s2i_image_name, + thoth_s2i_image_version=thoth_s2i_image_version, + is_external=False, + ) + ) + self._messages_logged.clear() + _LOGGER.debug("Analyzed image has the following symbols: %r", self.image_symbols) + super().pre_run() + + def run(self, package_versions: Generator[PackageVersion, None, None]) -> Generator[PackageVersion, None, None]: + """If package requires non-present symbols remove it.""" + if not self.image_symbols: + # No image symbols or the version was not provided. + return + + for pkg_vers in package_versions: + package_symbols = set( + self.context.graph.get_python_package_required_symbols( + package_name=pkg_vers.name, + package_version=pkg_vers.locked_version, + index_url=pkg_vers.index.url, + ) + ) + + # Shortcut if package requires no symbols + if not package_symbols: + yield pkg_vers + continue + + missing_symbols = package_symbols - self.image_symbols + if not missing_symbols: + yield pkg_vers + else: + # Log removed package + package_tuple = pkg_vers.to_tuple() + if package_tuple not in self._messages_logged: + message = f"Package {package_tuple} was removed due to missing ABI symbols in the environment" + _LOGGER.warning("%s - see %s", message, self._LINK) + self._messages_logged.add(package_tuple) + _LOGGER.debug("The following symbols are not present: %r", str(missing_symbols)) + + continue