Cleanup boto patching and include min/max (#3939)

* Cleanup boto patching and include min/max

scripts/boto/_automated_patches.py

@@ -4,13 +4,17 @@
 """
 
 import json
+import logging
 from pathlib import Path
 from typing import Any
 
+from _helpers import load_schema_file
 from _types import AllPatches, Patch, ResourcePatches
 
 from cfnlint.schema.resolver import RefResolutionError, RefResolver
 
+LOGGER = logging.getLogger("cfnlint")
+
 skip = [
     "account",
     "chime",
@@ -33,8 +37,6 @@
 
 skip_property_names = ["State"]
 
-_fields = ["pattern", "enum"]
-
 _visited_paths = []
 
 
@@ -125,9 +127,6 @@ def _nested_arrays(
             resolver, schema_data, boto_data, array_shap_data, path, source
         )
     else:
-        # skip if we already have an enum or pattern
-        if any([schema_data.get(field) for field in _fields]):
-            return {}
         return {
             path: Patch(
                 source=source,
@@ -167,10 +166,6 @@ def _nested_objects(
                     except RefResolutionError:
                         return results
 
-                # skip if we already have an enum or pattern
-                if any([p_data.get(field) for field in _fields]):
-                    continue
-
                 member_shape_name = member_data.get("shape")
                 member_shape = boto_data.get("shapes", {}).get(member_shape_name, {})
 
@@ -189,9 +184,6 @@ def _nested_objects(
                             )
                         )
 
-                if not any([member_shape.get(field) for field in _fields]):
-                    continue
-
                 results[path] = Patch(
                     source=source,
                     shape=member_shape_name,
@@ -201,13 +193,13 @@ def _nested_objects(
 
 
 def _per_resource_patch(
-    schema_data: dict[str, Any], boto_data: dict[str, Any], source: list[str]
+    resolver: RefResolver, boto_data: dict[str, Any], source: list[str]
 ) -> ResourcePatches:
     results: ResourcePatches = {}
+    _, schema_data = resolver.resolve("/")
     create_operations = get_schema_create_operations(schema_data)
     shapes = {}
 
-    resolver = RefResolver.from_schema(schema_data)
     for create_operation in create_operations:
         shapes.update(get_shapes(boto_data, create_operation))
         create_shape = (
@@ -252,21 +244,19 @@ def get_resource_patches(
 
     resources = list(schema_path.glob(f"aws-{service_name}-*.json"))
     if not resources:
-        print(f"No resource files found for {service_name}")
+        LOGGER.warning(f"No resource files found for {service_name}")
 
     for resource in resources:
-        with open(resource, "r") as f:
-            schema_data = json.load(f)
+        ref_resolver = load_schema_file(resource)
+        _, schema_data = ref_resolver.resolve("/")
 
-            resource_type = schema_data.get("typeName", "")
-            if resource_type not in results:
-                results[resource_type] = {}
+        resource_type = schema_data.get("typeName", "")
+        if resource_type not in results:
+            results[resource_type] = {}
 
-            results[resource_type].update(
-                _per_resource_patch(
-                    schema_data, boto_data, [service_dir.name, last_date]
-                )
-            )
+        results[resource_type].update(
+            _per_resource_patch(ref_resolver, boto_data, [service_dir.name, last_date])
+        )
 
     return results
 

scripts/boto/_helpers.py

@@ -0,0 +1,17 @@
+"""
+Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+SPDX-License-Identifier: MIT-0
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from cfnlint.schema.resolver import RefResolver
+
+
+def load_schema_file(schema_path: Path) -> RefResolver:
+    with open(schema_path, "r") as f:
+        data = json.load(f)
+        return RefResolver.from_schema(data)

scripts/boto/update_schemas_from_boto.py

@@ -14,6 +14,7 @@
 import regex as re
 import requests
 from _automated_patches import build_automated_patches
+from _helpers import load_schema_file
 from _manual_patches import patches
 from _types import AllPatches, ResourcePatches
 
@@ -39,7 +40,7 @@ def configure_logging():
     ch = logging.StreamHandler()
     ch.setLevel(logging.INFO)
 
-    LOGGER.setLevel(logging.INFO)
+    LOGGER.setLevel(logging.WARNING)
     log_formatter = logging.Formatter(
         "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
     )
@@ -52,10 +53,16 @@ def configure_logging():
 
 
 def build_resource_type_patches(
-    dir: str, resource_name: str, resource_patches: ResourcePatches
+    boto_path: Path,
+    schema_path: Path,
+    resource_name: str,
+    resource_patches: ResourcePatches,
 ):
     LOGGER.info(f"Applying patches for {resource_name}")
 
+    resolver = load_schema_file(
+        schema_path / f"{resource_name.lower().replace('::', '-')}.json"
+    )
     resource_name = resource_name.lower().replace("::", "_")
     output_path = Path("src/cfnlint/data/schemas/patches/extensions/all/")
     resource_path = output_path / resource_name
@@ -71,13 +78,38 @@ def build_resource_type_patches(
         service_path = (
             ["botocore-master/botocore/data"] + patch.source + ["service-2.json"]
         )
-        with open(os.path.join(dir, *service_path), "r") as f:
+        _, schema_data = resolver.resolve(f"#{path}")
+        with open(os.path.join(boto_path, *service_path), "r") as f:
             boto_d = json.load(f)
-
-            for field in ["enum", "pattern"]:
+            shape_type = boto_d.get("shapes", {}).get(patch.shape, {}).get("type")
+            for field in ["enum", "pattern", "max", "min"]:
                 value = boto_d.get("shapes", {}).get(patch.shape, {}).get(field)
                 if not value:
                     continue
+                if field in ["enum", "pattern"]:
+                    if any(f in schema_data for f in ["enum", "pattern"]):
+                        continue
+                elif field == "max":
+                    if any(
+                        f in schema_data for f in ["maxLength", "maxItems", "maximum"]
+                    ):
+                        continue
+                    if "pattern" in schema_data:
+                        if re.match(
+                            r"^.*\{[0-9]+,[0-9]+\}\$?$", schema_data["pattern"]
+                        ):
+                            continue
+
+                elif field == "min":
+                    if any(
+                        f in schema_data for f in ["minLength", "minItems", "minimum"]
+                    ):
+                        continue
+                    if "pattern" in schema_data:
+                        if re.match(
+                            r"^.*\{[0-9]+,[0-9]+\}\$?$", schema_data["pattern"]
+                        ):
+                            continue
                 if field == "pattern":
                     if value in [".*", "^.*$"]:
                         continue
@@ -92,6 +124,15 @@ def build_resource_type_patches(
                             )
                         )
                         continue
+                if field in ["max", "min"]:
+                    if shape_type == "string":
+                        field = "maxLength" if field == "max" else "minLength"
+                    elif shape_type == "list":
+                        field = "maxItems" if field == "max" else "minItems"
+                    elif shape_type in ["integer", "number"]:
+                        field = "maximum" if field == "max" else "minimum"
+                    else:
+                        continue
                 if value:
                     if patch.source[0] in exceptions:
                         if path in exceptions[patch.source[0]]:
@@ -123,12 +164,13 @@ def build_resource_type_patches(
 
 
 def build_patches(
-    dir: str,
+    boto_path: Path,
+    schema_path: Path,
     patches: AllPatches,
 ):
     for resource_name, patch in patches.items():
         build_resource_type_patches(
-            dir, resource_name=resource_name, resource_patches=patch
+            boto_path, schema_path, resource_name=resource_name, resource_patches=patch
         )
 
 
@@ -159,7 +201,7 @@ def main():
                     else:
                         _patches[k][path] = patch
 
-        build_patches(boto_path, _patches)
+        build_patches(boto_path, schema_path, _patches)
 
 
 if __name__ == "__main__":

src/cfnlint/data/DownloadsMetadata/123ba181485ae293d5bd09722af0c19d5a0d14c62111ff864923fc7b7960dda6.meta.json

@@ -1 +1 @@
-{"etag": "\"b7df0a841e1d85eb2ca55fae9b63a199\"", "url": "https://schema.cloudformation.eu-south-1.amazonaws.com/CloudformationSchema.zip"}
+{"etag": "\"3b5d6ce396c9a98b72b7fc1127bc1abc\"", "url": "https://schema.cloudformation.eu-south-1.amazonaws.com/CloudformationSchema.zip"}

src/cfnlint/data/DownloadsMetadata/18624fcc4a1c571691d10b2508e6be565e4752bbc10d9552de8df8f81348c42b.meta.json

@@ -1 +1 @@
-{"etag": "\"75e66d980dea62bf67981ffe569bbb47\"", "url": "https://schema.cloudformation.us-gov-east-1.amazonaws.com/CloudformationSchema.zip"}
+{"etag": "\"40c9314343c1df29f23241aca99ff5c0\"", "url": "https://schema.cloudformation.us-gov-east-1.amazonaws.com/CloudformationSchema.zip"}

src/cfnlint/data/DownloadsMetadata/23be708e675cb6098b08969e4dbbc3f54cfc32461d10e077e7e5de1fc25d8b8f.meta.json

@@ -1 +1 @@
-{"etag": "\"b4ea48ef051d441d6fe68349a08772e5\"", "url": "https://schema.cloudformation.us-gov-west-1.amazonaws.com/CloudformationSchema.zip"}
+{"etag": "\"4d01654834ffd168b5611579eb8746bc\"", "url": "https://schema.cloudformation.us-gov-west-1.amazonaws.com/CloudformationSchema.zip"}

src/cfnlint/data/DownloadsMetadata/26cf4521b683d3267844178a6bcd1d0ad3fe2e7838c23f6acd054944cb2b1618.meta.json

@@ -1 +1 @@
-{"etag": "\"6d426c7baa8bf4703793694e05b7699a\"", "url": "https://schema.cloudformation.me-central-1.amazonaws.com/CloudformationSchema.zip"}
+{"etag": "\"c0cee52c664bec22aa9cad20452c8be4\"", "url": "https://schema.cloudformation.me-central-1.amazonaws.com/CloudformationSchema.zip"}

src/cfnlint/data/DownloadsMetadata/276cecfeb1ec5e608e2aaa06925a2da57e9907e4a512c10ddae70b98b4dada43.meta.json

@@ -1 +1 @@
-{"etag": "\"4eff125b6a8d9a53bfd01cd720058a77\"", "url": "https://schema.cloudformation.eu-west-2.amazonaws.com/CloudformationSchema.zip"}
+{"etag": "\"704f1b32b8ed2357b4efeb93a9633e36\"", "url": "https://schema.cloudformation.eu-west-2.amazonaws.com/CloudformationSchema.zip"}

src/cfnlint/data/DownloadsMetadata/42155835f382d82337be3d2bf832bada376aa3fe15197e3bb0e9290ff8719b4e.meta.json

@@ -1 +1 @@
-{"etag": "\"b44fc7ac1956ad50020dcc9bce2f7e65\"", "url": "https://schema.cloudformation.af-south-1.amazonaws.com/CloudformationSchema.zip"}
+{"etag": "\"5d90eede497d94dc5acbe3947c384f0f\"", "url": "https://schema.cloudformation.af-south-1.amazonaws.com/CloudformationSchema.zip"}

src/cfnlint/data/DownloadsMetadata/42e9df95722b6524cd001503b6750b86f60a7b5acfc406ebb10d5748cbb8ed41.meta.json

@@ -1 +1 @@
-{"etag": "\"cccc338e071c166a131d1cdc9d74fab7\"", "url": "https://schema.cloudformation.us-west-1.amazonaws.com/CloudformationSchema.zip"}
+{"etag": "\"11cd6603e5e474388338daeea4ec18c5\"", "url": "https://schema.cloudformation.us-west-1.amazonaws.com/CloudformationSchema.zip"}

src/cfnlint/data/DownloadsMetadata/4fbb29b69678acdd32c5758ed43ead9bf35136af536e92a84ccbaf062c315066.meta.json

@@ -1 +1 @@
-{"etag": "\"2a5f184af1076cf4909dd7dfd66c1091\"", "url": "https://schema.cloudformation.eu-central-1.amazonaws.com/CloudformationSchema.zip"}
+{"etag": "\"f1245202dc7d03e32366cf183a60f5bf\"", "url": "https://schema.cloudformation.eu-central-1.amazonaws.com/CloudformationSchema.zip"}

src/cfnlint/data/DownloadsMetadata/56584c7d00e444de640bef01fc2c630804470222e5e4c690bacef5312891581d.meta.json

@@ -1 +1 @@
-{"etag": "\"81e2ca053c22e5c8e6f33990c3c1683a\"", "url": "https://schema.cloudformation.ap-south-1.amazonaws.com/CloudformationSchema.zip"}
+{"etag": "\"be46d4992c4b8b9d9363ed0f29d67bfc\"", "url": "https://schema.cloudformation.ap-south-1.amazonaws.com/CloudformationSchema.zip"}

src/cfnlint/data/DownloadsMetadata/583721567eccd1d5855daa623819df1a646b563d773e34b020d0ddeab2fe195b.meta.json

@@ -1 +1 @@
-{"etag": "\"5b4a791385a5b8095875163b14a84e12\"", "url": "https://schema.cloudformation.ap-southeast-4.amazonaws.com/CloudformationSchema.zip"}
+{"etag": "\"8b09581590dc797e7f8b9966df079bcd\"", "url": "https://schema.cloudformation.ap-southeast-4.amazonaws.com/CloudformationSchema.zip"}

src/cfnlint/data/DownloadsMetadata/6316ae24f21cb620947aa250bebbee69548d44cc32e246ec9d7742088a2c17f8.meta.json

@@ -1 +1 @@
-{"etag": "\"735d724f69e6c702c966f1bd667266dc\"", "url": "https://schema.cloudformation.us-east-2.amazonaws.com/CloudformationSchema.zip"}
+{"etag": "\"416452f27037f3b97f425284a73bcf9c\"", "url": "https://schema.cloudformation.us-east-2.amazonaws.com/CloudformationSchema.zip"}

src/cfnlint/data/DownloadsMetadata/81e1cc73ff2daf7d1e1eca393c2d1fdd98ac34d4109512e0e0947ef752dcb9c9.meta.json

@@ -1 +1 @@
-{"etag": "\"d05bdba3e6c8b67d31218a116456c75b\"", "url": "https://schema.cloudformation.ap-southeast-1.amazonaws.com/CloudformationSchema.zip"}
+{"etag": "\"b5126a7d839bd20951fcafe4da854d38\"", "url": "https://schema.cloudformation.ap-southeast-1.amazonaws.com/CloudformationSchema.zip"}

src/cfnlint/data/DownloadsMetadata/8b8b0cee4df1ef0947a8289e8ec0c67869b7533eabab32ecfc0a00cb19e55a5f.meta.json

@@ -1 +1 @@
-{"etag": "\"26a4a4a9660b28ecb31a40aa4db8b791\"", "url": "https://schema.cloudformation.ap-southeast-3.amazonaws.com/CloudformationSchema.zip"}
+{"etag": "\"6edf94664df234608592965308b6f6b7\"", "url": "https://schema.cloudformation.ap-southeast-3.amazonaws.com/CloudformationSchema.zip"}

src/cfnlint/data/DownloadsMetadata/b1f069477cd577cde04dfe1b323c0bb0e783fe32b6bb6b13774c32fcca1d303a.meta.json

@@ -1 +1 @@
-{"etag": "\"d37369e7d40a074400df68c04fb64d28\"", "url": "https://schema.cloudformation.ap-east-1.amazonaws.com/CloudformationSchema.zip"}
+{"etag": "\"174ddf0e8bbf351c26704e4e2337b16a\"", "url": "https://schema.cloudformation.ap-east-1.amazonaws.com/CloudformationSchema.zip"}

src/cfnlint/data/DownloadsMetadata/c7ada205073390b33b7593ef8f304b9705f2567698dfdfa979bf0ccdb68cb856.meta.json

@@ -1 +1 @@
-{"etag": "\"1faad9cae8146a0781efc270a2cf2ed3\"", "url": "https://schema.cloudformation.sa-east-1.amazonaws.com/CloudformationSchema.zip"}
+{"etag": "\"3e62904c961fae6ced725b12783ef45d\"", "url": "https://schema.cloudformation.sa-east-1.amazonaws.com/CloudformationSchema.zip"}

src/cfnlint/data/DownloadsMetadata/d24ce9a45a014b1ff04d479422ea956e92030ae5c03d7451980a15735e557edb.meta.json

@@ -1 +1 @@
-{"etag": "\"5da9230365aca3d094395f17b18cb452\"", "url": "https://schema.cloudformation.ap-southeast-2.amazonaws.com/CloudformationSchema.zip"}
+{"etag": "\"83baf7b0a3b0ee8c5f5ba852405d693e\"", "url": "https://schema.cloudformation.ap-southeast-2.amazonaws.com/CloudformationSchema.zip"}

src/cfnlint/data/DownloadsMetadata/dd98171253ebc36f5b78e247f3132b5f25c8d66a1f84939600616bab42579541.meta.json

@@ -1 +1 @@
-{"etag": "\"2b0a5723c1a5e739eea87299385382f9\"", "url": "https://schema.cloudformation.eu-north-1.amazonaws.com/CloudformationSchema.zip"}
+{"etag": "\"e292cb731d4b0b927a8d86a6301127ae\"", "url": "https://schema.cloudformation.eu-north-1.amazonaws.com/CloudformationSchema.zip"}

src/cfnlint/data/DownloadsMetadata/df4945435575c690a2651bb31e7a9b48972142778e1ff452383ede2ad4bac3d7.meta.json

@@ -1 +1 @@
-{"etag": "\"260acbfe6a05bd388bccf119e9747069\"", "url": "https://schema.cloudformation.eu-south-2.amazonaws.com/CloudformationSchema.zip"}
+{"etag": "\"c75fbe544296584ef4d56b592fb87ed3\"", "url": "https://schema.cloudformation.eu-south-2.amazonaws.com/CloudformationSchema.zip"}

src/cfnlint/data/DownloadsMetadata/e5301e07e25fa2c35d2c7e1f9dcf720826b315ef6f38515840537c02de23abe2.meta.json

@@ -1 +1 @@
-{"etag": "\"10d837e83b0dee3b5bdd8025d2e030f2\"", "url": "https://schema.cloudformation.ca-central-1.amazonaws.com/CloudformationSchema.zip"}
+{"etag": "\"485c9eb6f7e1efdb2af30a9800b05fab\"", "url": "https://schema.cloudformation.ca-central-1.amazonaws.com/CloudformationSchema.zip"}

src/cfnlint/data/DownloadsMetadata/e8b3dacc1675b478e8c7392b51f41467cf908a34e6b4c3fb3e97e2b584f651ca.meta.json

@@ -1 +1 @@
-{"etag": "\"316784e70cf0510fea4b81b31ecd8f15\"", "url": "https://schema.cloudformation.eu-west-3.amazonaws.com/CloudformationSchema.zip"}
+{"etag": "\"8fc65b5326dd82fb678168c25661e666\"", "url": "https://schema.cloudformation.eu-west-3.amazonaws.com/CloudformationSchema.zip"}

src/cfnlint/data/DownloadsMetadata/ea0f7b8f144feb225afe73a24dfdf993318c41c71c21b0a17d4f68d82c5aee21.meta.json

@@ -1 +1 @@
-{"etag": "\"b4ec2276c5d96d861a33de8d7ef645f4\"", "url": "https://schema.cloudformation.ap-northeast-3.amazonaws.com/CloudformationSchema.zip"}
+{"etag": "\"d89138fadb813a9999c4f07c383b48e7\"", "url": "https://schema.cloudformation.ap-northeast-3.amazonaws.com/CloudformationSchema.zip"}

src/cfnlint/data/DownloadsMetadata/f1896c9151984eec294af1eddf64260f6cd7e4ced378cacdb93f76ed227b5c5d.meta.json

@@ -1 +1 @@
-{"etag": "\"47806c7293decd2a8dc0b6e7ec30b75f\"", "url": "https://schema.cloudformation.us-west-2.amazonaws.com/CloudformationSchema.zip"}
+{"etag": "\"dd30b5012826298b344563720602a94b\"", "url": "https://schema.cloudformation.us-west-2.amazonaws.com/CloudformationSchema.zip"}

src/cfnlint/data/DownloadsMetadata/f49718b210ea89ff182ae51cb7004366b9e2e4d5e38136a5be83b6a55e7a82a1.meta.json

@@ -1 +1 @@
-{"etag": "\"aec4bdf0cc84ce7f3a8486d83c3b41de\"", "url": "https://schema.cloudformation.ap-south-2.amazonaws.com/CloudformationSchema.zip"}
+{"etag": "\"707ec23d98096f232e1f68967ce937bd\"", "url": "https://schema.cloudformation.ap-south-2.amazonaws.com/CloudformationSchema.zip"}

src/cfnlint/data/DownloadsMetadata/f54eee6f8ad9619f41835b700369cdbb41c64a9c91b2fa5b4928c0d9b2f780b0.meta.json

@@ -1 +1 @@
-{"etag": "\"6e1fd9ae967709a2a2069c40a2db3aa2\"", "url": "https://schema.cloudformation.us-east-1.amazonaws.com/CloudformationSchema.zip"}
+{"etag": "\"91387a48563a194dd6a2d7b943bd8005\"", "url": "https://schema.cloudformation.us-east-1.amazonaws.com/CloudformationSchema.zip"}

src/cfnlint/data/DownloadsMetadata/f6f35a459759d6c132fa2fe798cefbd5b2d398fe33547eed780b9b70f10eb4a2.meta.json

@@ -1 +1 @@
-{"etag": "\"854a88a3967badd9ca15fab91e861ad3\"", "url": "https://schema.cloudformation.il-central-1.amazonaws.com/CloudformationSchema.zip"}
+{"etag": "\"0ff66b9ec501bfa3f5235ad1b46b6901\"", "url": "https://schema.cloudformation.il-central-1.amazonaws.com/CloudformationSchema.zip"}

src/cfnlint/data/DownloadsMetadata/ff02b7d808c1c00053f09aa43a50addf3b69878d351cffd417dc9a457df808af.meta.json

@@ -1 +1 @@
-{"etag": "\"cee4b474888719e8896f94090ac6e16d\"", "url": "https://schema.cloudformation.ap-northeast-1.amazonaws.com/CloudformationSchema.zip"}
+{"etag": "\"76471ec41ead11d47f95ac6393542b12\"", "url": "https://schema.cloudformation.ap-northeast-1.amazonaws.com/CloudformationSchema.zip"}

src/cfnlint/data/schemas/patches/extensions/all/aws_accessanalyzer_analyzer/boto.json

@@ -18,5 +18,15 @@
   "op": "add",
   "path": "/definitions/ArchiveRule/properties/RuleName/pattern",
   "value": "[A-Za-z][A-Za-z0-9_.-]*"
+ },
+ {
+  "op": "add",
+  "path": "/definitions/ArchiveRule/properties/RuleName/maxLength",
+  "value": 255
+ },
+ {
+  "op": "add",
+  "path": "/definitions/ArchiveRule/properties/RuleName/minLength",
+  "value": 1
  }
 ]