Filter out packages based on ABI symbols present in S2I Thoth

tests/sieves/test_thoth_s2i_abi_compat.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+# thoth-adviser
+# Copyright(C) 2020 Kevin Postlethwait
+#
+# This program is free software: you can redistribute it and / or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""Test filtering out Python Packages based on required and available ABI symbols."""
+
+from typing import Optional
+
+import flexmock
+import pytest
+
+from thoth.adviser.context import Context
+from thoth.adviser.enums import RecommendationType
+from thoth.adviser.pipeline_builder import PipelineBuilderContext
+from thoth.adviser.sieves import ThothS2IAbiCompatibilitySieve
+from thoth.python import PackageVersion
+from thoth.python import Source
+from thoth.storages import GraphDatabase
+
+from ..base import AdviserUnitTestCase
+
+_SYSTEM_SYMBOLS = ["GLIBC_2.0", "GLIBC_2.1", "GLIBC_2.2", "GLIBC_2.3", "GLIBC_2.4", "GLIBC_2.5", "GCC_3.4", "X_2.21"]
+_REQUIRED_SYMBOLS_A = ["GLIBC_2.9"]
+_REQUIRED_SYMBOLS_B = ["GLIBC_2.4"]
+
+
+class TestThothS2IAbiCompatibilitySieve(AdviserUnitTestCase):
+    """Test filtering out packages based on symbols required."""
+
+    UNIT_TESTED = ThothS2IAbiCompatibilitySieve
+
+    def test_verify_multiple_should_include(self, builder_context: PipelineBuilderContext) -> None:
+        """Verify multiple should_include calls do not loop endlessly."""
+        builder_context.recommendation_type = RecommendationType.LATEST
+        builder_context.project.runtime_environment.base_image = "quay.io/thoth-station/s2i-thoth-ubi8-py38:v0.23.0"
+        self.verify_multiple_should_include(builder_context)
+
+    @pytest.mark.parametrize("base_image", [
+        None,
+        "fedora:32",
+    ])
+    def test_no_should_include(self, base_image: Optional[str], builder_context: PipelineBuilderContext) -> None:
+        """Test not including this pipeline unit."""
+        builder_context.project.runtime_environment.base_image = base_image
+        assert self.UNIT_TESTED.should_include(builder_context) is None
+
+    def test_should_include(self, builder_context: PipelineBuilderContext) -> None:
+        """Test including this pipeline unit."""
+        builder_context.project.runtime_environment.base_image = "quay.io/thoth-station/s2i-thoth-ubi8-py38:v0.23.0"
+        assert self.UNIT_TESTED.should_include(builder_context) == {}
+
+    def test_no_thoth_s2i_version(self, context: Context) -> None:
+        """Test no Thoth S2I version present."""
+        context.project.runtime_environment.base_image = "quay.io/thoth-station/s2i-thoth-ubi8-py38"  # No version.
+
+        GraphDatabase.should_receive("get_thoth_s2i_analyzed_image_symbols_all").times(0)
+
+        unit = self.UNIT_TESTED()
+        with self.UNIT_TESTED.assigned_context(context):
+            unit.pre_run()
+
+        assert not unit.image_symbols
+
+    def test_abi_compat_symbols_present(self, context: Context) -> None:
+        """Test if required symbols are correctly identified."""
+        source = Source("https://pypi.org/simple")
+        package_version = PackageVersion(name="tensorflow", version="==1.9.0", index=source, develop=False)
+        flexmock(GraphDatabase)
+        GraphDatabase.should_receive("get_thoth_s2i_analyzed_image_symbols_all").and_return(_SYSTEM_SYMBOLS).once()
+        GraphDatabase.should_receive("get_python_package_required_symbols").and_return(_REQUIRED_SYMBOLS_B).once()
+
+        context.project.runtime_environment = flexmock(
+            operating_system=flexmock(name="rhel", version="8.0"),
+            cuda_version="4.6",
+            python_version="3.6",
+        )
+
+        with self.UNIT_TESTED.assigned_context(context):
+            sieve = self.UNIT_TESTED()
+            sieve.pre_run()
+            assert list(sieve.run((p for p in [package_version]))) == [package_version]
+
+    def test_abi_compat_symbols_not_present(self, context: Context) -> None:
+        """Test if required symbols being missing is correctly identified."""
+        source = Source("https://pypi.org/simple")
+        package_version = PackageVersion(name="tensorflow", version="==1.9.0", index=source, develop=False)
+        flexmock(GraphDatabase)
+        GraphDatabase.should_receive("get_thoth_s2i_analyzed_image_symbols_all").and_return(_SYSTEM_SYMBOLS).once()
+        GraphDatabase.should_receive("get_python_package_required_symbols").and_return(_REQUIRED_SYMBOLS_A).once()
+
+        context.project.runtime_environment = flexmock(
+            operating_system=flexmock(name="rhel", version="8.0"),
+            cuda_version="4.6",
+            python_version="3.6",
+        )
+
+        with self.UNIT_TESTED.assigned_context(context):
+            sieve = self.UNIT_TESTED()
+            sieve.pre_run()
+            assert list(sieve.run((p for p in [package_version]))) == []
+
+    def test_super_pre_run(self, context: Context) -> None:
+        """Make sure the pre-run method of the base is called."""
+        context.graph.should_receive("get_analyzed_image_symbols_all").with_args(
+            os_name=context.project.runtime_environment.operating_system.name,
+            os_version=context.project.runtime_environment.operating_system.version,
+            cuda_version=context.project.runtime_environment.cuda_version,
+            python_version=context.project.runtime_environment.python_version,
+        ).and_return(set()).once()
+
+        unit = self.UNIT_TESTED()
+        assert unit.unit_run is False
+
+        with unit.assigned_context(context):
+            unit.pre_run()
+
+        assert unit.unit_run is False

thoth/adviser/sieves/thoth_s2i_abi_compat.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+# thoth-adviser
+# Copyright(C) 2019-2021 Kevin Postlehtwait, Fridolin Pokorny
+#
+# This program is free software: you can redistribute it and / or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""Filter out stacks which have require non-existent ABI symbols in Thoth's s2i base image."""
+
+import logging
+from typing import Any, Dict, Optional, Generator, Set, TYPE_CHECKING, Tuple
+
+import attr
+from thoth.common import get_justification_link as jl
+from thoth.python import PackageVersion
+
+from ..sieve import Sieve
+
+if TYPE_CHECKING:
+    from ..pipeline_builder import PipelineBuilderContext
+
+_LOGGER = logging.getLogger(__name__)
+
+
+@attr.s(slots=True)
+class ThothS2IAbiCompatibilitySieve(Sieve):
+    """Remove packages if the Thoth's s2i image being used doesn't have necessary ABI."""
+
+    _THOTH_S2I_PREFIX = "quay.io/thoth-station/"
+    CONFIGURATION_DEFAULT = {"package_name": None}
+    image_symbols = attr.ib(type=Set[str], factory=set, init=False)
+    _messages_logged = attr.ib(type=Set[Tuple[str, str, str]], factory=set, init=False)
+
+    _LINK = jl("abi_missing")
+    _LINK_BAD_IMAGE = jl("bad_base_image")
+
+    @classmethod
+    def should_include(cls, builder_context: "PipelineBuilderContext") -> Optional[Dict[str, Any]]:
+        """Register if the base image provided is Thoth's s2i."""
+        if builder_context.is_included(cls):
+            return None
+
+        base_image = builder_context.project.runtime_environment.base_image
+        if base_image and base_image.startswith(cls._THOTH_S2I_PREFIX):
+            return {}
+
+        return None
+
+    def pre_run(self) -> None:
+        """Initialize image_symbols."""
+        base_image = self.context.project.runtime_environment.base_image
+        parts = base_image.split(":", maxsplit=1)
+        if len(parts) != 2:
+            error_msg = f"Cannot determine Thoth s2i version information from {base_image}, "\
+                        "recommendations specific for ABI used will not be taken into account"
+            _LOGGER.warning("%s - see %s", error_msg, self._LINK_BAD_IMAGE)
+            self.context.stack_info.append({
+                "type": "WARNING",
+                "message": error_msg,
+                "link": self._LINK_BAD_IMAGE,
+            })
+
+            self.image_symbols = set()
+            return
+
+        thoth_s2i_image_name, thoth_s2i_image_version = parts
+        if thoth_s2i_image_version.startswith("v"):
+            # Not nice as we always prefix with "v" but do not store it with "v" in the database
+            # (based on env var exported and detected in Thoth's s2i).
+            thoth_s2i_image_version = thoth_s2i_image_version[1:]
+
+        self.image_symbols = set(
+            self.context.graph.get_thoth_s2i_analyzed_image_symbols_all(
+                thoth_s2i_image_name=thoth_s2i_image_name,
+                thoth_s2i_image_version=thoth_s2i_image_version,
+                is_external=False,
+            )
+        )
+        self._messages_logged.clear()
+        _LOGGER.debug("Analyzed image has the following symbols: %r", self.image_symbols)
+        super().pre_run()
+
+    def run(self, package_versions: Generator[PackageVersion, None, None]) -> Generator[PackageVersion, None, None]:
+        """If package requires non-present symbols remove it."""
+        if not self.image_symbols:
+            # No image symbols or the version was not provided.
+            return
+
+        for pkg_vers in package_versions:
+            package_symbols = set(
+                self.context.graph.get_python_package_required_symbols(
+                    package_name=pkg_vers.name,
+                    package_version=pkg_vers.locked_version,
+                    index_url=pkg_vers.index.url,
+                )
+            )
+
+            # Shortcut if package requires no symbols
+            if not package_symbols:
+                yield pkg_vers
+                continue
+
+            missing_symbols = package_symbols - self.image_symbols
+            if not missing_symbols:
+                yield pkg_vers
+            else:
+                # Log removed package
+                package_tuple = pkg_vers.to_tuple()
+                if package_tuple not in self._messages_logged:
+                    message = f"Package {package_tuple} was removed due to missing ABI symbols in the environment"
+                    _LOGGER.warning("%s - see %s", message, self._LINK)
+                    self._messages_logged.add(package_tuple)
+                    _LOGGER.debug("The following symbols are not present: %r", str(missing_symbols))
+
+                continue