[FEATURE] Add parsing of Expectation diagnostics to contrib packaging JSON object

contrib/cli/great_expectations_contrib/cli.py

@@ -6,7 +6,7 @@
     init_cmd,
     publish_cmd,
     read_package_from_file,
-    write_package_to_disk,
+    sync_package,
 )
 from great_expectations_contrib.package import GreatExpectationsContribPackageManifest
 
@@ -39,16 +39,20 @@ def init() -> None:
 @click.pass_obj
 def publish(pkg: GreatExpectationsContribPackageManifest) -> None:
     publish_cmd()
-    pkg.update_package_state()
-    write_package_to_disk(pkg, PACKAGE_PATH)
+    sync_package(pkg, PACKAGE_PATH)
 
 
 @cli.command(help="Check your package to make sure it's met all the requirements")
 @click.pass_obj
 def check(pkg: GreatExpectationsContribPackageManifest) -> None:
     check_cmd()
-    pkg.update_package_state()
-    write_package_to_disk(pkg, PACKAGE_PATH)
+    sync_package(pkg, PACKAGE_PATH)
+
+
+@cli.command(help="Manually sync your package state")
+@click.pass_obj
+def sync(pkg: GreatExpectationsContribPackageManifest) -> None:
+    sync_package(pkg, PACKAGE_PATH)
 
 
 if __name__ == "__main__":

contrib/cli/great_expectations_contrib/commands.py

@@ -162,7 +162,7 @@ def read_package_from_file(path: str) -> GreatExpectationsContribPackageManifest
         contents = f.read()
 
     data = json.loads(contents)
-    logger.info(f"Succesfully read existing package data from {path}")
+    logger.info(f"Successfully read existing package data from {path}")
     return GreatExpectationsContribPackageManifest(**data)
 
 
@@ -176,11 +176,17 @@ def write_package_to_disk(
         path: The relative path to the target package JSON file.
     """
     json_dict = asdict(package)
-    to_delete = [key for key, val in json_dict.items() if val is None]
-    for key in to_delete:
-        del json_dict[key]
-
     data = json.dumps(json_dict, indent=4)
     with open(path, "w") as f:
         f.write(data)
         logger.info(f"Succesfully wrote state to {path}.")
+
+
+def sync_package(package: GreatExpectationsContribPackageManifest, path: str) -> None:
+    """Evaluate the state of the contributor package and update the existing manifest.
+
+    Args:
+        package: The GreatExpectationsContribPackageManifest you wish to update/sync.
+    """
+    package.update_package_state()
+    write_package_to_disk(package, path)

contrib/cli/great_expectations_contrib/package.py

@@ -5,89 +5,92 @@
 import sys
 from dataclasses import dataclass
 from enum import Enum
-from typing import Any, Dict, List, Optional, Type
+from typing import Any, List, Optional, Type
 
+import pkg_resources
+
+from great_expectations.core.expectation_diagnostics.expectation_diagnostics import (
+    ExpectationDiagnostics,
+)
 from great_expectations.expectations.expectation import Expectation
+from great_expectations.types import SerializableDictDot
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
 
-# Type alias that will need to be updated to reflect the complex nature of the 'run_diagnostics' return object
-Diagnostics = Dict[str, Any]
-
 
-@dataclass(frozen=True)
-class ExpectationCompletenessCheck:
+@dataclass
+class ExpectationCompletenessCheck(SerializableDictDot):
     message: str
     passed: bool
 
 
-@dataclass(frozen=True)
-class ExpectationCompletenessChecklist:
+@dataclass
+class ExpectationCompletenessChecklist(SerializableDictDot):
     experimental: List[ExpectationCompletenessCheck]
     beta: List[ExpectationCompletenessCheck]
     production: List[ExpectationCompletenessCheck]
 
 
-@dataclass(frozen=True)
-class PackageCompletenessStatus:
+@dataclass
+class PackageCompletenessStatus(SerializableDictDot):
     concept_only: int
     experimental: int
     beta: int
     production: int
     total: int
 
 
-@dataclass(frozen=True)
-class RenderedExpectation:
+@dataclass
+class RenderedExpectation(SerializableDictDot):
     name: str
     tags: List[str]
     supported: List[str]
     status: ExpectationCompletenessChecklist
 
 
-@dataclass(frozen=True)
-class Dependency:
+@dataclass
+class Dependency(SerializableDictDot):
     text: str
     link: str
-    version: Optional[str]
+    version: Optional[str] = None
 
 
-@dataclass(frozen=True)
-class GitHubUser:
+@dataclass
+class GitHubUser(SerializableDictDot):
     username: str
-    full_name: Optional[str]
+    full_name: Optional[str] = None
 
 
-class SocialLinkType(Enum):
+class SocialLinkType(str, Enum):
     TWITTER = "TWITTER"
     INSTAGRAM = "INSTAGRAM"
     LINKEDIN = "LINKEDIN"
     MEDIUM = "MEDIUM"
 
 
-@dataclass(frozen=True)
-class SocialLink:
+@dataclass
+class SocialLink(SerializableDictDot):
     account_type: SocialLinkType
     identifier: str
 
 
-@dataclass(frozen=True)
-class DomainExpert:
+@dataclass
+class DomainExpert(SerializableDictDot):
     full_name: str
     social_links: List[SocialLink]
     picture: str
 
 
-class Maturity(Enum):
+class Maturity(str, Enum):
     CONCEPT_ONLY = "CONCEPT_ONLY"
     EXPERIMENTAL = "EXPERIMENTAL"
     BETA = "BETA"
     PRODUCTION = "PRODUCTION"
 
 
-@dataclass(frozen=True)
-class GreatExpectationsContribPackageManifest:
+@dataclass
+class GreatExpectationsContribPackageManifest(SerializableDictDot):
     # Core
     package_name: Optional[str] = None
     icon: Optional[str] = None
@@ -110,22 +113,97 @@ def update_package_state(self) -> None:
         """
         Parses diagnostic reports from package Expectations and uses them to update JSON state
         """
-        diagnostics = self._retrieve_package_expectations_diagnostics()
+        diagnostics = (
+            GreatExpectationsContribPackageManifest.retrieve_package_expectations_diagnostics()
+        )
         self._update_attrs_with_diagnostics(diagnostics)
 
-    def _update_attrs_with_diagnostics(self, diagnostics: List[Diagnostics]) -> None:
-        # TODO: Write logic to assign values to attrs
-        # This is a black box for now
-        # for diagnostic in diagnostics:
-        #     pass
-        raise NotImplementedError
-
-    def _retrieve_package_expectations_diagnostics(self) -> List[Diagnostics]:
+    def _update_attrs_with_diagnostics(
+        self, diagnostics: List[ExpectationDiagnostics]
+    ) -> None:
+        self._update_expectations(diagnostics)
+        self._update_dependencies("requirements.txt")
+        self._update_contributors(diagnostics)
+
+    def _update_expectations(self, diagnostics: List[ExpectationDiagnostics]) -> None:
+        expectations = []
+        status = {maturity.name: 0 for maturity in Maturity}
+
+        for diagnostic in diagnostics:
+            expectation = RenderedExpectation(
+                name=diagnostic.description.snake_name,
+                tags=diagnostic.library_metadata.tags,
+                supported=[],
+                status=diagnostic.maturity_checklist,  # Should be converted to the proper type
+            )
+            expectations.append(expectation)
+
+            expectation_maturity = diagnostic.library_metadata.maturity
+            status[expectation_maturity] += 1
+
+        self.expectations = expectations
+        self.expectation_count = len(expectations)
+
+        # Enum is all caps but status attributes are lowercase
+        lowercase_status = {k.lower(): v for k, v in status.items()}
+        lowercase_status["total"] = sum(status.values())
+
+        self.status = PackageCompletenessStatus(**lowercase_status)
+        maturity = max(status, key=status.get)
+        self.maturity = Maturity[maturity]
+
+    def _update_dependencies(self, path: str) -> None:
+        if not os.path.exists(path):
+            logger.warning(f"Could not find requirements file {path}")
+            self.dependencies = []
+            return
+
+        with open(path) as f:
+            requirements = [req for req in pkg_resources.parse_requirements(f)]
+
+        def _convert_to_dependency(
+            requirement: pkg_resources.Requirement,
+        ) -> Dependency:
+            name = requirement.project_name
+            pypi_url = f"https://pypi.org/project/{name}"
+            if requirement.specs:
+                # Stringify tuple of pins
+                version = ", ".join(
+                    "".join(symbol for symbol in pin)
+                    for pin in sorted(requirement.specs)
+                )
+            else:
+                version = None
+            return Dependency(text=name, link=pypi_url, version=version)
+
+        dependencies = list(map(_convert_to_dependency, requirements))
+        self.dependencies = dependencies
+
+    def _update_contributors(self, diagnostics: List[ExpectationDiagnostics]) -> None:
+        contributors = []
+        for diagnostic in diagnostics:
+            for contributor in diagnostic.library_metadata.contributors:
+                github_user = GitHubUser(contributor)
+                if github_user not in contributors:
+                    contributors.append(github_user)
+
+        self.contributors = contributors
+
+    @staticmethod
+    def retrieve_package_expectations_diagnostics() -> List[ExpectationDiagnostics]:
         try:
-            package = self._identify_user_package()
-            expectations_module = self._import_expectations_module(package)
-            expectations = self._retrieve_expectations_from_module(expectations_module)
-            diagnostics = self._gather_diagnostics(expectations)
+            package = GreatExpectationsContribPackageManifest._identify_user_package()
+            expectations_module = (
+                GreatExpectationsContribPackageManifest._import_expectations_module(
+                    package
+                )
+            )
+            expectations = GreatExpectationsContribPackageManifest._retrieve_expectations_from_module(
+                expectations_module
+            )
+            diagnostics = GreatExpectationsContribPackageManifest._gather_diagnostics(
+                expectations
+            )
             return diagnostics
         except Exception as e:
             # Exceptions should not break the CLI - this behavior should be working in the background
@@ -135,7 +213,8 @@ def _retrieve_package_expectations_diagnostics(self) -> List[Diagnostics]:
             )
             return []
 
-    def _identify_user_package(self) -> str:
+    @staticmethod
+    def _identify_user_package() -> str:
         # Guaranteed to have a dir named '<MY_PACKAGE>_expectations' through Cookiecutter validation
         packages = [
             d for d in os.listdir() if os.path.isdir(d) and d.endswith("_expectations")
@@ -149,7 +228,8 @@ def _identify_user_package(self) -> str:
 
         return packages[0]
 
-    def _import_expectations_module(self, package: str) -> Any:
+    @staticmethod
+    def _import_expectations_module(package: str) -> Any:
         # Need to add user's project to the PYTHONPATH
         cwd = os.getcwd()
         sys.path.append(cwd)
@@ -159,8 +239,9 @@ def _import_expectations_module(self, package: str) -> Any:
         except ModuleNotFoundError:
             raise
 
+    @staticmethod
     def _retrieve_expectations_from_module(
-        self, expectations_module: Any
+        expectations_module: Any,
     ) -> List[Type[Expectation]]:
         expectations: List[Type[Expectation]] = []
         names: List[str] = []
@@ -172,9 +253,10 @@ def _retrieve_expectations_from_module(
         logger.info(f"Found {len(names)} expectation(s): {names}")
         return expectations
 
+    @staticmethod
     def _gather_diagnostics(
-        self, expectations: List[Type[Expectation]]
-    ) -> List[Diagnostics]:
+        expectations: List[Type[Expectation]],
+    ) -> List[ExpectationDiagnostics]:
         diagnostics_list = []
         for expectation in expectations:
             instance = expectation()