c7n_tencentcloud - tests - support vcr for flight recording (cloud-cu…

…stodian#7735)

.github/workflows/ci-main.yml

@@ -208,7 +208,7 @@ jobs:
             . test.env && poetry run pytest -n auto tests tools \
               --cov c7n --cov tools/c7n_azure/c7n_azure \
               --cov tools/c7n_gcp/c7n_gcp --cov tools/c7n_kube/c7n_kube \
-              --cov tools/c7n_mailer/c7n_mailer
+              --cov tools/c7n_mailer/c7n_mailer --cov tools/c7n_tencentcloud/c7n_tencentcloud
             poetry run coverage xml
           else
             . test.env && poetry run pytest -n auto tests tools

pyproject.toml

@@ -51,6 +51,7 @@ twine = "^3.1.1"
 pytest-sugar = "^0.9.2"
 click = "^8.0"
 freezegun = "^1.2.2"
+pytest-recording = "^0.12.1"
 
 [tool.black]
 skip-string-normalization = true

test.env

@@ -10,3 +10,5 @@ export AZURE_TENANT_ID=ea42f556-5106-4743-99b0-000000000000
 export GOOGLE_CLOUD_PROJECT=custodian-1291
 export GOOGLE_APPLICATION_CREDENTIALS=tools/c7n_gcp/tests/data/credentials.json
 export C7N_VALIDATE=true
+export TENCENTCLOUD_SECRET_ID=AiIDOaWCckbC000000000000000000000000
+export TENCENTCLOUD_SECRET_KEY=qQxmZ5JG000000000000000000000000

tests/conftest.py

@@ -26,6 +26,8 @@ class LazyPluginCacheDir:
         pass
 
 
+pytest_plugins = ("pytest_recording",)
+
 # If we have C7N_FUNCTIONAL make sure Replay is False otherwise enable Replay
 LazyReplay.value = not strtobool(os.environ.get('C7N_FUNCTIONAL', 'no'))
 LazyPluginCacheDir.value = '../.tfcache'

tools/c7n_tencentcloud/c7n_tencentcloud/client.py

@@ -72,11 +72,13 @@ def execute_paged_query(self, action: str, params: dict,
 
         if paging_method == PageMethod.Offset:
             params[PageMethod.Offset.name] = 0
+            params[paging_def["limit"]["key"]] = paging_def["limit"]["value"]
         elif paging_method == PageMethod.PaginationToken:
             params[PageMethod.PaginationToken.name] = ""
             pagination_token_path = paging_def.get("pagination_token_path", "")
             if not pagination_token_path:
                 raise PolicyExecutionError("config to use pagination_token but not set token path")
+            params[paging_def["limit"]["key"]] = paging_def["limit"]["value"]
         else:
             raise PolicyExecutionError("unsupported paging method")
 
@@ -92,7 +94,8 @@ def execute_paged_query(self, action: str, params: dict,
             if len(items) > 0:
                 results.extend(items)
                 if paging_method == PageMethod.Offset:
-                    params[PageMethod.Offset.name] = params[PageMethod.Offset.name] + 1
+                    params[PageMethod.Offset.name] = params[PageMethod.Offset.name] +\
+                        paging_def["limit"]["value"]
                 else:
                     token = jmespath.search(pagination_token_path, result)
                     if token == "":

tools/c7n_tencentcloud/c7n_tencentcloud/query.py

@@ -36,7 +36,6 @@ class ResourceTypeInfo(metaclass=TypeMeta):
     # enum_spec: ("action", "jsonpath", "extra_params")
     enum_spec = ()
     paging_def: dir = {}  # define how to do paging
-    page_number_path: str = ""
     batch_size: int = 10
 
     # used by metric filter
@@ -225,7 +224,12 @@ def resources(self, params=None):
         if params is None:
             params = {}
 
-        res = self.query_helper.filter(self.resource_manager.config.region,
+        if self.resource_manager.resource_type.paging_def:
+            res = self.query_helper.paged_filter(self.resource_manager.config.region,
+                                            self.resource_manager.resource_type,
+                                            params)
+        else:
+            res = self.query_helper.filter(self.resource_manager.config.region,
                                        self.resource_manager.resource_type,
                                        params)
         self.augment(res)
@@ -246,7 +250,7 @@ def get_resource_tag(self, resource):
         """
         result = []
         resouce_id = resource[self.resource_type.id]
-        ce6 = self.get_resource6(resouce_id)
+        ce6 = self.get_resource_qcs(resouce_id)
         res = self.query_helper.get_resource_tags(self.region, ce6)
         if res:
             # get the resource's tags
@@ -257,9 +261,9 @@ def get_resource_tag(self, resource):
                 })
         return result
 
-    def get_resource6(self, resource_id):
+    def get_resource_qcs(self, resource_id):
         """
-        get_resource6
+        get_resource_qcs
         resource description https://cloud.tencent.com/document/product/598/10606
         """
         # qcs::${ServiceType}:${Region}:${Account}:${ResourcePreifx}/${ResourceId}

tools/c7n_tencentcloud/c7n_tencentcloud/resources/cvm.py

@@ -3,6 +3,7 @@
 
 from c7n_tencentcloud.provider import resources
 from c7n_tencentcloud.query import ResourceTypeInfo, QueryResourceManager
+from c7n_tencentcloud.utils import PageMethod
 
 
 @resources.register("cvm")
@@ -14,5 +15,7 @@ class resource_type(ResourceTypeInfo):
         endpoint = "cvm.tencentcloudapi.com"
         service = "cvm"
         version = "2017-03-12"
-        enum_spec = ("DescribeInstances", "Response.InstanceSet[]", None)
+        enum_spec = ("DescribeInstances", "Response.InstanceSet[]", {})
+        metrics_instance_id_name = "InstanceId"
+        paging_def = {"method": PageMethod.Offset, "limit": {"Key": "Limit", "value": 20}}
         resource_preifx = "instance"

tools/c7n_tencentcloud/c7n_tencentcloud/resources/tag.py

@@ -18,13 +18,15 @@ class resource_type(ResourceTypeInfo):
         taggable = True
         enum_spec = ("GetResources",
                      "Response.ResourceTagMappingList[]",
-                     {
-                         "PaginationToken": "",
-                         "MaxResults": 20
-                     })
+                     {}
+                     )
         paging_def = {
             "method": PageMethod.PaginationToken,
-            "pagination_token_path": "Response.PaginationToken"
+            "pagination_token_path": "Response.PaginationToken",
+            "limit": {
+                "Key": "MaxResults",
+                "value": 200
+            }
         }
 
     def get_resource_query_params(self):

tools/c7n_tencentcloud/pyproject.toml

@@ -16,6 +16,7 @@ pytest = "^7.1.2"
 c7n = {path = "../..", develop = true}
 freezegun = "^1.2.2"
 pytest-cov = "^3.0.0"
+pytest-recording = "^0.12.1"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

tools/c7n_tencentcloud/requirements.txt

@@ -1,9 +1,9 @@
 certifi==2022.6.15; python_version >= "3.7" and python_version < "4"
-charset-normalizer>=2.1.0; python_version >= "3.7" and python_version < "4" and python_full_version >= "3.6.0"
+charset-normalizer==2.1.0; python_version >= "3.7" and python_version < "4" and python_full_version >= "3.6.0"
 idna==3.3; python_version >= "3.7" and python_version < "4"
-pytz>=2022.1
+pytz==2022.1
 requests==2.28.1; python_version >= "3.7" and python_version < "4"
 retrying==1.3.3
 six==1.16.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.3.0"
-tencentcloud-sdk-python==3.0.717
-urllib3>=1.26.11; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.7"
+tencentcloud-sdk-python==3.0.727
+urllib3==1.26.11; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.7"

tools/c7n_tencentcloud/setup.py

@@ -1,5 +1,3 @@
-# Copyright The Cloud Custodian Authors.
-# SPDX-License-Identifier: Apache-2.0
 # Automatically generated from poetry/pyproject.toml
 # flake8: noqa
 # -*- coding: utf-8 -*-
@@ -17,13 +15,15 @@
 install_requires = \
 ['argcomplete (>=2.0.0,<3.0.0)',
  'attrs (>=22.1.0,<23.0.0)',
- 'boto3 (>=1.24.64,<2.0.0)',
- 'botocore (>=1.27.64,<2.0.0)',
+ 'boto3 (>=1.24.65,<2.0.0)',
+ 'botocore (>=1.27.65,<2.0.0)',
  'c7n (>=0.9.18,<0.10.0)',
  'docutils (>=0.17.1,<0.18.0)',
  'importlib-metadata (>=4.12.0,<5.0.0)',
+ 'importlib-resources (>=5.9.0,<6.0.0)',
  'jmespath (>=1.0.1,<2.0.0)',
  'jsonschema (>=4.15.0,<5.0.0)',
+ 'pkgutil-resolve-name (>=1.3.10,<2.0.0)',
  'pyrsistent (>=0.18.1,<0.19.0)',
  'python-dateutil (>=2.8.2,<3.0.0)',
  'pytz>=2022.1',
@@ -33,6 +33,7 @@
  'six (>=1.16.0,<2.0.0)',
  'tabulate (>=0.8.10,<0.9.0)',
  'tencentcloud-sdk-python>=3.0.715,<4.0.0',
+ 'typing-extensions (>=4.3.0,<5.0.0)',
  'urllib3 (>=1.26.12,<2.0.0)',
  'zipp (>=3.8.1,<4.0.0)']
 
@@ -46,17 +47,17 @@
         'Topic :: System :: Systems Administration',
         'Topic :: System :: Distributed Computing'
     ],
-    'long_description': None,
+    'long_description': 'None',
     'long_description_content_type': 'text/markdown',
     'author': 'Tencent Cloud',
-    'author_email': None,
-    'maintainer': None,
-    'maintainer_email': None,
-    'url': None,
+    'author_email': 'None',
+    'maintainer': 'None',
+    'maintainer_email': 'None',
+    'url': 'None',
     'packages': packages,
     'package_data': package_data,
     'install_requires': install_requires,
-    'python_requires': '>=3.9,<4.0',
+    'python_requires': '>=3.7,<4.0',
 }