Merge commit '02110441487f2dcd3808d23d1f34cfe7a8fb41b8' into merge-up…

…stream

.codecov.yml

@@ -15,20 +15,23 @@ coverage:
         only_pulls: true
     project:
       azure:
-        paths: "tools/c7n_azure"
+        paths: ["tools/c7n_azure"]
         threshold: "1%"
       gcp:
-        paths: "tools/c7n_gcp"
+        paths: ["tools/c7n_gcp"]
         threshold: "1%"
       mailer:
-        paths: "tools/c7n_mailer"
+        paths: ["tools/c7n_mailer"]
         threshold: "3%"
       k8s:
-        paths: "tools/c7n_kube"
+        paths: ["tools/c7n_kube"]
         threshold: "1%"
       tf:
-        paths: "tools/c7n_terraform"
+        paths: ["tools/c7n_terraform"]
         threshold: "1%"
+      shiftleft:
+        paths: ["tools/c7n_left"]
+        threshold: "3%"
       # aws is weaved throughout core atm can't easily call it out separately
       custodian:
       # one of the tests is slightly variable coverage due to concurrency (0.01)

.github/workflows/ci-main.yml

@@ -22,10 +22,13 @@ jobs:
       - name: Install Linter
         run: |
           python -m pip install --upgrade pip
-          pip install flake8==4.0.1
+          pip install flake8 black
       - name: Lint Check
         run: |
           make lint
+      - name: Format Check
+        run: |
+          black --check tools/c7n_left
 
   Analyzer:
     runs-on: ubuntu-latest
@@ -75,8 +78,7 @@ jobs:
       - name: Bootstrap poetry
         shell: bash
         run: |
-          curl -sL https://raw.githubusercontent.com/python-poetry/poetry/master/install-poetry.py \
-            | python - -y --version $POETRY_VERSION
+          curl -sSL https://raw.githubusercontent.com/python-poetry/install.python-poetry.org/6161821b1d39fa30f92a677bba51abfc471f8aee/install-poetry.py | python3 - --version $POETRY_VERSION -y
 
       - name: Set up cache
         uses: actions/cache@v2
@@ -128,23 +130,25 @@ jobs:
       - name: Test
         shell: bash
         env:
-          COV_RUN: ${{ contains(matrix.python-version, '3.8') && contains(matrix.os, 'ubuntu') }}
+          COV_RUN: ${{ contains(matrix.python-version, '3.10') && contains(matrix.os, 'ubuntu') }}
         run: |
           if [[ "$COV_RUN" == "true" ]]
           then
             echo "Running Coverage Test"
             . test.env && poetry run pytest -n auto tests tools \
               --cov c7n --cov tools/c7n_azure/c7n_azure \
               --cov tools/c7n_gcp/c7n_gcp --cov tools/c7n_kube/c7n_kube \
-              --cov tools/c7n_mailer/c7n_mailer --cov tools/c7n_tencentcloud/c7n_tencentcloud
+              --cov tools/c7n_tencentcloud/c7n_tencentcloud \
+              --cov tools/c7n_left/c7n_left \
+              --cov tools/c7n_mailer/c7n_mailer
             poetry run coverage xml
           else
             . test.env && poetry run pytest -n auto tests tools
           fi
 
       - name: Upload Code Coverage
         uses: codecov/codecov-action@v2
-        if: contains(matrix.python-version, '3.8') && contains(matrix.os, 'ubuntu')
+        if: contains(matrix.python-version, '3.10') && contains(matrix.os, 'ubuntu')
         with:
           files: ./coverage.xml
           name: codecov

Makefile

@@ -1,6 +1,18 @@
 SELF_MAKE := $(lastword $(MAKEFILE_LIST))
 PKG_REPO = testpypi
-PKG_SET = tools/c7n_gcp tools/c7n_kube tools/c7n_openstack tools/c7n_mailer tools/c7n_logexporter tools/c7n_policystream tools/c7n_trailcreator tools/c7n_org tools/c7n_sphinxext tools/c7n_terraform tools/c7n_awscc tools/c7n_tencentcloud tools/c7n_azure
+PKG_SET := tools/c7n_gcp tools/c7n_kube tools/c7n_openstack tools/c7n_mailer tools/c7n_logexporter tools/c7n_policystream tools/c7n_trailcreator tools/c7n_org tools/c7n_sphinxext tools/c7n_terraform tools/c7n_awscc tools/c7n_tencentcloud tools/c7n_azure
+
+PLATFORM_ARCH := $(shell python3 -c "import platform; print(platform.machine())")
+PLATFORM_OS := $(shell python3 -c "import platform; print(platform.system())")
+PY_VERSION := $(shell python3 -c "import sys; print('%s.%s' % (sys.version_info.major, sys.version_info.minor))")
+
+
+ifneq "$(findstring $(PLATFORM_OS), Linux Darwin)" ""
+  ifneq "$(findstring $(PY_VERSION), 3.10)" ""
+    PKG_SET := tools/c7n_left $(PKG_SET)
+  endif
+endif
+
 
 install:
 	python3 -m venv .

c7n/loader.py

@@ -185,8 +185,9 @@ def _validate(data):
                 log.error("Configuration invalid: {}".format(data))
                 log.error("%s" % e)
                 errors.append(e)
-            load_resources(structure.get_resource_types(data))
-            schm = schema.generate()
+            rtypes = structure.get_resource_types(data)
+            load_resources(rtypes)
+            schm = schema.generate(rtypes)
             errors += schema.validate(data, schm)
             return errors
 

c7n/policy.py

@@ -24,6 +24,7 @@
 from c7n import deprecated, utils
 from c7n.version import version
 from c7n.query import RetryPageIterator
+from c7n.varfmt import VarFormat
 
 log = logging.getLogger('c7n.policy')
 
@@ -1132,7 +1133,9 @@ def resource_type(self) -> str:
 
     @property
     def provider_name(self) -> str:
-        if '.' in self.resource_type:
+        if isinstance(self.resource_type, list):
+            provider_name, _ = self.resource_type[0].split('.', 1)
+        elif '.' in self.resource_type:
             provider_name, resource_type = self.resource_type.split('.', 1)
         else:
             provider_name = 'aws'
@@ -1243,7 +1246,9 @@ def expand_variables(self, variables):
         Updates the policy data in-place.
         """
         # format string values returns a copy
-        updated = utils.format_string_values(self.data, **variables)
+        var_fmt = VarFormat()
+        updated = utils.format_string_values(
+            self.data, formatter=var_fmt.format, **variables)
 
         # Several keys should only be expanded at runtime, perserve them.
         if 'member-role' in updated.get('mode', {}):

c7n/provider.py

@@ -69,7 +69,10 @@ def import_resource_classes(resource_map, resource_types):
         if r not in resource_map:
             not_found.add(r)
             continue
-        rmodule, rclass = resource_map[r].rsplit('.', 1)
+        provider_value = resource_map[r]
+        if isinstance(provider_value, type):
+            continue
+        rmodule, rclass = provider_value.rsplit('.', 1)
         rmods.add(rmodule)
 
     for rmodule in rmods:
@@ -78,6 +81,10 @@ def import_resource_classes(resource_map, resource_types):
     for rtype in resource_types:
         if rtype in not_found:
             continue
+        provider_value = resource_map[rtype]
+        if isinstance(provider_value, type):
+            found.append(provider_value)
+            continue
         rmodule, rclass = resource_map[rtype].rsplit('.', 1)
         r = getattr(mod_map[rmodule], rclass, None)
         if r is None:
@@ -102,6 +109,8 @@ def resources(cloud_provider=None):
 
 
 def get_resource_class(resource_type):
+    if isinstance(resource_type, list):
+        resource_type = resource_type[0]
     if '.' in resource_type:
         provider_name, resource = resource_type.split('.', 1)
     else:

c7n/resources/__init__.py

@@ -28,16 +28,16 @@ def load_resources(resource_types=('*',)):
     return missing
 
 
-def should_load_provider(name, provider_types):
+def should_load_provider(name, provider_types, no_wild=False):
     global LOADED
     if (name not in LOADED and
-        ('*' in provider_types or
-         name in provider_types)):
+        (('*' in provider_types and not no_wild)
+         or name in provider_types)):
         return True
     return False
 
 
-PROVIDER_NAMES = ('aws', 'azure', 'gcp', 'k8s', 'openstack', 'awscc', 'tencentcloud')
+PROVIDER_NAMES = ('aws', 'azure', 'gcp', 'k8s', 'openstack', 'awscc', 'tencentcloud', 'terraform')
 
 
 def load_available(resources=True):
@@ -93,6 +93,10 @@ def load_providers(provider_types):
         from c7n_tencentcloud.entry import initialize_tencentcloud
         initialize_tencentcloud()
 
+    if should_load_provider('terraform', provider_types, no_wild=True):
+        from c7n_left.entry import initialize_iac
+        initialize_iac()
+
     if should_load_provider('c7n', provider_types):
         from c7n import data  # noqa