Add a sieve to filter out packages based on ABI provided by s2i Thoth

tests/sieves/test_thoth_s2i_abi_compat.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+# thoth-adviser
+# Copyright(C) 2020-2021 Kevin Postlethwait
+#
+# This program is free software: you can redistribute it and / or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""Test filtering out Python Packages based on required and available ABI symbols."""
+
+from typing import Optional
+
+import flexmock
+import pytest
+
+from thoth.adviser.context import Context
+from thoth.adviser.enums import RecommendationType
+from thoth.adviser.pipeline_builder import PipelineBuilderContext
+from thoth.adviser.sieves import ThothS2IAbiCompatibilitySieve
+from thoth.python import PackageVersion
+from thoth.python import Source
+from thoth.storages import GraphDatabase
+
+from ..base import AdviserUnitTestCase
+
+_SYSTEM_SYMBOLS = ["GLIBC_2.0", "GLIBC_2.1", "GLIBC_2.2", "GLIBC_2.3", "GLIBC_2.4", "GLIBC_2.5", "GCC_3.4", "X_2.21"]
+_REQUIRED_SYMBOLS_A = ["GLIBC_2.9"]
+_REQUIRED_SYMBOLS_B = ["GLIBC_2.4"]
+
+
+class TestThothS2IAbiCompatibilitySieve(AdviserUnitTestCase):
+    """Test filtering out packages based on symbols required."""
+
+    UNIT_TESTED = ThothS2IAbiCompatibilitySieve
+
+    def test_verify_multiple_should_include(self, builder_context: PipelineBuilderContext) -> None:
+        """Verify multiple should_include calls do not loop endlessly."""
+        builder_context.recommendation_type = RecommendationType.LATEST
+        builder_context.project.runtime_environment.base_image = "quay.io/thoth-station/s2i-thoth-ubi8-py38:v0.23.0"
+        self.verify_multiple_should_include(builder_context)
+
+    @pytest.mark.parametrize(
+        "base_image",
+        [
+            None,
+            "fedora:32",
+        ],
+    )
+    def test_no_should_include(self, base_image: Optional[str], builder_context: PipelineBuilderContext) -> None:
+        """Test not including this pipeline unit."""
+        builder_context.project.runtime_environment.base_image = base_image
+        assert self.UNIT_TESTED.should_include(builder_context) is None
+
+    def test_should_include(self, builder_context: PipelineBuilderContext) -> None:
+        """Test including this pipeline unit."""
+        builder_context.project.runtime_environment.base_image = "quay.io/thoth-station/s2i-thoth-ubi8-py38:v0.23.0"
+        assert self.UNIT_TESTED.should_include(builder_context) == {}
+
+    def test_no_thoth_s2i_version(self, context: Context) -> None:
+        """Test no Thoth S2I version present."""
+        context.project.runtime_environment.base_image = "quay.io/thoth-station/s2i-thoth-ubi8-py38"  # No version.
+
+        GraphDatabase.should_receive("get_thoth_s2i_analyzed_image_symbols_all").times(0)
+
+        unit = self.UNIT_TESTED()
+        with self.UNIT_TESTED.assigned_context(context):
+            unit.pre_run()
+
+        assert not unit.image_symbols
+
+    def test_abi_compat_symbols_present(self, context: Context) -> None:
+        """Test if required symbols are correctly identified."""
+        source = Source("https://pypi.org/simple")
+        package_version = PackageVersion(name="tensorflow", version="==1.9.0", index=source, develop=False)
+        flexmock(GraphDatabase)
+        GraphDatabase.should_receive("get_thoth_s2i_analyzed_image_symbols_all").and_return(_SYSTEM_SYMBOLS).once()
+        GraphDatabase.should_receive("get_python_package_required_symbols").and_return(_REQUIRED_SYMBOLS_B).once()
+
+        context.project.runtime_environment.operating_system.name = "rhel"
+        context.project.runtime_environment.operating_system.version = "8.0"
+        context.project.runtime_environment.cuda_version = "4.6"
+        context.project.runtime_environment.python_version = "3.6"
+        context.project.runtime_environment.base_image = "quay.io/thoth-station/s2i-thoth-ubi8-py38:v0.23.0"
+
+        with self.UNIT_TESTED.assigned_context(context):
+            sieve = self.UNIT_TESTED()
+            sieve.pre_run()
+            assert list(sieve.run((p for p in [package_version]))) == [package_version]
+
+    def test_abi_compat_symbols_not_present(self, context: Context) -> None:
+        """Test if required symbols being missing is correctly identified."""
+        source = Source("https://pypi.org/simple")
+        package_version = PackageVersion(name="tensorflow", version="==1.9.0", index=source, develop=False)
+        flexmock(GraphDatabase)
+        GraphDatabase.should_receive("get_thoth_s2i_analyzed_image_symbols_all").and_return(_SYSTEM_SYMBOLS).once()
+        GraphDatabase.should_receive("get_python_package_required_symbols").and_return(_REQUIRED_SYMBOLS_A).once()
+
+        context.project.runtime_environment.operating_system.name = "rhel"
+        context.project.runtime_environment.operating_system.version = "8.0"
+        context.project.runtime_environment.cuda_version = "4.6"
+        context.project.runtime_environment.python_version = "3.6"
+        context.project.runtime_environment.base_image = "quay.io/thoth-station/s2i-thoth-ubi8-py38:v0.23.0"
+
+        with self.UNIT_TESTED.assigned_context(context):
+            sieve = self.UNIT_TESTED()
+            sieve.pre_run()
+            assert list(sieve.run((p for p in [package_version]))) == []
+
+    def test_super_pre_run(self, context: Context) -> None:
+        """Make sure the pre-run method of the base is called."""
+        context.graph.should_receive("get_thoth_s2i_analyzed_image_symbols_all").with_args(
+            thoth_s2i_image_name="quay.io/thoth-station/s2i-thoth-ubi8-py38",
+            thoth_s2i_image_version="0.23.0",
+            is_external=False,
+        ).and_return(set()).once()
+
+        context.project.runtime_environment.base_image = "quay.io/thoth-station/s2i-thoth-ubi8-py38:v0.23.0"
+
+        unit = self.UNIT_TESTED()
+        assert unit.unit_run is False
+
+        with unit.assigned_context(context):
+            unit.pre_run()
+
+        assert unit.unit_run is False

thoth/adviser/sieves/__init__.py

@@ -17,7 +17,6 @@
 
 """Implementation of sieves used in adviser pipeline."""
 
-from .abi_compat import AbiCompatibilitySieve
 from .backports import Enum34BackportSieve
 from .backports import Functools32BackportSieve
 from .backports import ImportlibMetadataBackportSieve
@@ -34,6 +33,7 @@
 from .tensorflow import TensorFlowAPISieve
 from .tensorflow import TensorFlowCUDASieve
 from .tensorflow import TensorFlowPython39Sieve
+from .thoth_s2i_abi_compat import ThothS2IAbiCompatibilitySieve
 from .version_constraint import VersionConstraintSieve
 
 
@@ -46,7 +46,7 @@
     "PackageIndexSieve",
     "SolvedSieve",
     "VersionConstraintSieve",
-    "AbiCompatibilitySieve",
+    "ThothS2IAbiCompatibilitySieve",
     "FilterIndexSieve",
     "Enum34BackportSieve",
     "Functools32BackportSieve",

thoth/adviser/sieves/abi_compat.py → thoth/adviser/sieves/thoth_s2i_abi_compat.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # thoth-adviser
-# Copyright(C) 2019, 2020 Kevin Postlehtwait
+# Copyright(C) 2019-2021 Kevin Postlehtwait
 #
 # This program is free software: you can redistribute it and / or modify
 # it under the terms of the GNU General Public License as published by
@@ -15,7 +15,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 
-"""Filter out stacks which have require non-existent ABI symbols."""
+"""Filter out stacks which have require non-existent ABI symbols in Thoth's s2i base image."""
 
 import logging
 from typing import Any, Dict, Optional, Generator, Set, TYPE_CHECKING, Tuple
@@ -33,32 +33,61 @@
 
 
 @attr.s(slots=True)
-class AbiCompatibilitySieve(Sieve):
-    """Remove packages if the image being used doesn't have necessary ABI."""
+class ThothS2IAbiCompatibilitySieve(Sieve):
+    """Remove packages if the Thoth's s2i image being used doesn't have necessary ABI."""
 
+    _THOTH_S2I_PREFIX = "quay.io/thoth-station/"
     CONFIGURATION_DEFAULT = {"package_name": None}
     image_symbols = attr.ib(type=Set[str], factory=set, init=False)
     _messages_logged = attr.ib(type=Set[Tuple[str, str, str]], factory=set, init=False)
 
     _LINK = jl("abi_missing")
+    _LINK_BAD_IMAGE = jl("bad_base_image")
 
     @classmethod
     def should_include(cls, builder_context: "PipelineBuilderContext") -> Optional[Dict[str, Any]]:
-        """Include sieve which checks for abi compatability."""
-        if not builder_context.is_included(cls) and builder_context.project.runtime_environment.is_fully_specified():
+        """Register if the base image provided is Thoth's s2i."""
+        if builder_context.is_included(cls):
+            return None
+
+        base_image = builder_context.project.runtime_environment.base_image
+        if base_image and base_image.startswith(cls._THOTH_S2I_PREFIX):
             return {}
 
         return None
 
     def pre_run(self) -> None:
         """Initialize image_symbols."""
-        runtime_environment = self.context.project.runtime_environment
+        base_image = self.context.project.runtime_environment.base_image
+        parts = base_image.split(":", maxsplit=1)
+        if len(parts) != 2:
+            error_msg = (
+                f"Cannot determine Thoth s2i version information from {base_image}, "
+                "recommendations specific for ABI used will not be taken into account"
+            )
+            _LOGGER.warning("%s - see %s", error_msg, self._LINK_BAD_IMAGE)
+            self.context.stack_info.append(
+                {
+                    "type": "WARNING",
+                    "message": error_msg,
+                    "link": self._LINK_BAD_IMAGE,
+                }
+            )
+
+            self.image_symbols = set()
+            return
+
+        thoth_s2i_image_name, thoth_s2i_image_version = parts
+        if thoth_s2i_image_version.startswith("v"):
+            # Not nice as we always prefix with "v" but do not store it with "v" in the database
+            # (based on env var exported and detected in Thoth's s2i).
+            thoth_s2i_image_version = thoth_s2i_image_version[1:]
+
         self.image_symbols = set(
-            self.context.graph.get_analyzed_image_symbols_all(
-                os_name=runtime_environment.operating_system.name,
-                os_version=runtime_environment.operating_system.version,
-                cuda_version=runtime_environment.cuda_version,
-                python_version=runtime_environment.python_version,
+            self.context.graph.get_thoth_s2i_analyzed_image_symbols_all(
+                thoth_s2i_image_name=thoth_s2i_image_name,
+                thoth_s2i_image_version=thoth_s2i_image_version,
+                is_external=False,
             )
         )
         self._messages_logged.clear()
@@ -67,6 +96,10 @@ def pre_run(self) -> None:
 
     def run(self, package_versions: Generator[PackageVersion, None, None]) -> Generator[PackageVersion, None, None]:
         """If package requires non-present symbols remove it."""
+        if not self.image_symbols:
+            # No image symbols or the version was not provided.
+            return
+
         for pkg_vers in package_versions:
             package_symbols = set(
                 self.context.graph.get_python_package_required_symbols(