[FEATURE] Add parsing of Expectation diagnostics to contrib packaging JSON object

assets/scripts/build_package_gallery.py

@@ -0,0 +1,94 @@
+# Purpose: Aggregate all contrib packages into a single JSON file to populate the gallery
+#
+# The generated file is sent to S3 through our CI/CD to be rendered on the front-end.
+
+
+import json
+import logging
+import os
+from dataclasses import asdict
+from typing import List
+
+from contrib.cli.great_expectations_contrib.commands import (
+    read_package_from_file,
+    sync_package,
+)
+from contrib.cli.great_expectations_contrib.package import (
+    GreatExpectationsContribPackageManifest,
+)
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+
+def gather_all_contrib_package_paths() -> List[str]:
+    """Iterate through contrib/ and identify the relative paths to all contrib packages.
+
+    A contrib package is defined by the existence of a .great_expectations_package.json file.
+
+    Returns:
+        List of relative paths pointing to contrib packages
+    """
+    package_paths: List[str] = []
+    for root, _, files in os.walk("contrib/"):
+        for file in files:
+            if file == ".great_expectations_package.json":
+                package_paths.append(root)
+
+    logger.info(f"Found {len(package_paths)} contrib packages")
+    return package_paths
+
+
+def gather_all_package_manifests(package_paths: List[str]) -> List[dict]:
+    """Takes a list of relative paths to contrib packages and collects dictionaries to represent package state.
+
+    Args:
+        package_paths: A list of relative paths point to contrib packages
+
+    Returns:
+        A list of dictionaries that represents contributor package manifests
+    """
+    payload: List[dict] = []
+    root = os.getcwd()
+    for path in package_paths:
+        try:
+            # Go to package, read manifest, and sync it
+            os.chdir(path)
+            package: GreatExpectationsContribPackageManifest = read_package_from_file(
+                path
+            )
+            sync_package(package, path)
+
+            # Serialize to dict to append to payload
+            json_data: dict = asdict(package)
+            payload.append(json_data)
+            logger.info(
+                f"Successfully serialized {package.package_name} to dict and appended to manifest list"
+            )
+        except Exception as e:
+            logger.warning(
+                f"Something went wrong when syncing {path} and serializing to dict: {e}"
+            )
+        finally:
+            # Always ensure we revert back to the project root
+            os.chdir(root)
+
+    return payload
+
+
+def write_results_to_disk(path: str, package_manifests: List[dict]) -> None:
+    """Take the list of package manifests and write to JSON file.
+
+    Args:
+        path: The relative path to write to
+        package_manifest: A list of dictionaries that represents contributor package manifests
+    """
+    with open(path, "w") as outfile:
+        json.dump(package_manifests, outfile)
+        logger.info(f"Successfully wrote package manifests to {path}")
+
+
+if __name__ == "__main__":
+    package_paths = gather_all_contrib_package_paths()
+    payload = gather_all_package_manifests(package_paths)
+    write_results_to_disk("./package_manifests.json", payload)

contrib/cli/great_expectations_contrib/cli.py

@@ -6,7 +6,7 @@
     init_cmd,
     publish_cmd,
     read_package_from_file,
-    write_package_to_disk,
+    sync_package,
 )
 from great_expectations_contrib.package import GreatExpectationsContribPackageManifest
 
@@ -39,16 +39,20 @@ def init() -> None:
 @click.pass_obj
 def publish(pkg: GreatExpectationsContribPackageManifest) -> None:
     publish_cmd()
-    pkg.update_package_state()
-    write_package_to_disk(pkg, PACKAGE_PATH)
+    sync_package(pkg, PACKAGE_PATH)
 
 
 @cli.command(help="Check your package to make sure it's met all the requirements")
 @click.pass_obj
 def check(pkg: GreatExpectationsContribPackageManifest) -> None:
     check_cmd()
-    pkg.update_package_state()
-    write_package_to_disk(pkg, PACKAGE_PATH)
+    sync_package(pkg, PACKAGE_PATH)
+
+
+@cli.command(help="Manually sync your package state")
+@click.pass_obj
+def sync(pkg: GreatExpectationsContribPackageManifest) -> None:
+    sync_package(pkg, PACKAGE_PATH)
 
 
 if __name__ == "__main__":

contrib/cli/great_expectations_contrib/commands.py

@@ -162,7 +162,7 @@ def read_package_from_file(path: str) -> GreatExpectationsContribPackageManifest
         contents = f.read()
 
     data = json.loads(contents)
-    logger.info(f"Succesfully read existing package data from {path}")
+    logger.info(f"Successfully read existing package data from {path}")
     return GreatExpectationsContribPackageManifest(**data)
 
 
@@ -176,11 +176,17 @@ def write_package_to_disk(
         path: The relative path to the target package JSON file.
     """
     json_dict = asdict(package)
-    to_delete = [key for key, val in json_dict.items() if val is None]
-    for key in to_delete:
-        del json_dict[key]
-
     data = json.dumps(json_dict, indent=4)
     with open(path, "w") as f:
         f.write(data)
         logger.info(f"Succesfully wrote state to {path}.")
+
+
+def sync_package(package: GreatExpectationsContribPackageManifest, path: str) -> None:
+    """Evaluate the state of the contributor package and update the existing manifest.
+
+    Args:
+        package: The GreatExpectationsContribPackageManifest you wish to update/sync.
+    """
+    package.update_package_state()
+    write_package_to_disk(package, path)