sam pipeline bootstrap (aws#2811)

* two-stages-pipeline plugin * typos * add docstring * make mypy happy * removing swap file * delete the two_stages_pipeline plugin as the pipeline-bootstrap command took over its responsibility * remove 'get_template_function_runtimes' function as the decision is made to not process the SAM template during pipeline init which was the only place we use the function * sam pipeline bootstrap command * move the pipelineconfig.toml file to .aws-sam * UX - rewriting Co-authored-by: Chris Rehn <[email protected]> * UX improvements * make black happy * apply review comments * UX - rewriting Co-authored-by: Chris Rehn <[email protected]> * refactor * Apply review comments * use python way of array elements assignments * Update samcli/lib/pipeline/bootstrap/stage.py Co-authored-by: _sam <[email protected]> * apply review comments * typo * read using utf-8 * create and user a safe version of the save_config method * apply review comments * rename _get_command_name to _get_command_names * don't save generated ARNs for now, will save during init * Revert "don't save generated ARNs for now, will save during init" This reverts commit d184e16. * Notify the user to rotate periodically rotate the IAM credentials * typo * Use AES instead of KMS for S3 SSE * rename Ecr to ECR and Iam to IAM * Grant lambda service explicit permissions to thhe ECR instead of relying on giving this permissions on ad-hoc while creating the container images Co-authored-by: Chris Rehn <[email protected]> Co-authored-by: _sam <[email protected]>
aahung · May 26, 2021 · 89ef6d4 · 89ef6d4
1 parent 321caf5
commit 89ef6d4
Show file tree

Hide file tree

Showing 34 changed files with 2,069 additions and 60 deletions.
diff --git a/mypy.ini b/mypy.ini
@@ -59,6 +59,6 @@ ignore_missing_imports=True
 ignore_missing_imports=True
 
 # progressive add typechecks and these modules already complete the process, let's keep them clean
-[mypy-samcli.commands.build,samcli.lib.build.*,samcli.commands.local.cli_common.invoke_context,samcli.commands.local.lib.local_lambda,samcli.lib.providers.*,samcli.lib.utils.git_repo.py]
+[mypy-samcli.commands.build,samcli.lib.build.*,samcli.commands.local.cli_common.invoke_context,samcli.commands.local.lib.local_lambda,samcli.lib.providers.*,samcli.lib.utils.git_repo.py,samcli.lib.cookiecutter.*,samcli.lib.pipeline.*,samcli.commands.pipeline.*]
 disallow_untyped_defs=True
 disallow_incomplete_defs=True
diff --git a/samcli/cli/command.py b/samcli/cli/command.py
@@ -21,6 +21,7 @@
     "samcli.commands.deploy",
     "samcli.commands.logs",
     "samcli.commands.publish",
+    "samcli.commands.pipeline.pipeline",
     # We intentionally do not expose the `bootstrap` command for now. We might open it up later
     # "samcli.commands.bootstrap",
 ]

diff --git a/samcli/cli/context.py b/samcli/cli/context.py
@@ -4,7 +4,7 @@
 
 import logging
 import uuid
-from typing import Optional, cast
+from typing import Optional, cast, List
 
 import boto3
 import botocore
@@ -186,7 +186,7 @@ def _refresh_session(self):
             raise CredentialsError(str(ex)) from ex
 
 
-def get_cmd_names(cmd_name, ctx):
+def get_cmd_names(cmd_name, ctx) -> List[str]:
     """
     Given the click core context, return a list representing all the subcommands passed to the CLI
 

diff --git a/samcli/commands/_utils/template.py b/samcli/commands/_utils/template.py
@@ -9,16 +9,16 @@
 import yaml
 from botocore.utils import set_value_from_jmespath
 
-from samcli.commands.exceptions import UserException
-from samcli.lib.utils.packagetype import ZIP
-from samcli.yamlhelper import yaml_parse, yaml_dump
 from samcli.commands._utils.resources import (
     METADATA_WITH_LOCAL_PATHS,
     RESOURCES_WITH_LOCAL_PATHS,
     AWS_SERVERLESS_FUNCTION,
     AWS_LAMBDA_FUNCTION,
     get_packageable_resource_paths,
 )
+from samcli.commands.exceptions import UserException
+from samcli.lib.utils.packagetype import ZIP
+from samcli.yamlhelper import yaml_parse, yaml_dump
 
 
 class TemplateNotFoundException(UserException):

diff --git a/samcli/commands/pipeline/__init__.py b/samcli/commands/pipeline/__init__.py
diff --git a/samcli/commands/pipeline/bootstrap/__init__.py b/samcli/commands/pipeline/bootstrap/__init__.py
diff --git a/samcli/commands/pipeline/bootstrap/cli.py b/samcli/commands/pipeline/bootstrap/cli.py
@@ -0,0 +1,216 @@
+"""
+CLI command for "pipeline bootstrap", which sets up the require pipeline infrastructure resources
+"""
+import os
+from typing import Any, Dict, List, Optional
+
+import click
+
+from samcli.cli.cli_config_file import configuration_option, TomlProvider
+from samcli.cli.context import get_cmd_names
+from samcli.cli.main import pass_context, common_options, aws_creds_options, print_cmdline_args
+from samcli.lib.config.samconfig import SamConfig
+from samcli.lib.pipeline.bootstrap.stage import Stage
+from samcli.lib.telemetry.metric import track_command
+from samcli.lib.utils.version_checker import check_newer_version
+from .guided_context import GuidedContext
+
+SHORT_HELP = "Sets up infrastructure resources for AWS SAM CI/CD pipelines."
+
+HELP_TEXT = """Sets up the following infrastructure resources for AWS SAM CI/CD pipelines:
+\n\t - Pipeline IAM user with access key ID and secret access key credentials to be shared with the CI/CD provider
+\n\t - Pipeline execution IAM role assumed by the pipeline user to obtain access to the AWS account
+\n\t - CloudFormation execution IAM role assumed by CloudFormation to deploy the AWS SAM application
+\n\t - Artifacts S3 bucket to hold the AWS SAM build artifacts
+\n\t - Optionally, an ECR repository to hold container image Lambda deployment packages
+"""
+
+PIPELINE_CONFIG_DIR = os.path.join(".aws-sam", "pipeline")
+PIPELINE_CONFIG_FILENAME = "pipelineconfig.toml"
+
+
+@click.command("bootstrap", short_help=SHORT_HELP, help=HELP_TEXT, context_settings=dict(max_content_width=120))
+@configuration_option(provider=TomlProvider(section="parameters"))
+@click.option(
+    "--interactive/--no-interactive",
+    is_flag=True,
+    default=True,
+    help="Disable interactive prompting for bootstrap parameters, and fail if any required arguments are missing.",
+)
+@click.option(
+    "--stage-name",
+    help="The name of the corresponding pipeline stage. It is used as a suffix for the created resources.",
+    required=False,
+)
+@click.option(
+    "--pipeline-user",
+    help="The ARN of the IAM user having its access key ID and secret access key shared with the CI/CD provider. "
+    "It is used to grant this IAM user the permissions to access the corresponding AWS account. "
+    "If not provided, the command will create one along with access key ID and secret access key credentials.",
+    required=False,
+)
+@click.option(
+    "--pipeline-execution-role",
+    help="The ARN of an IAM role to be assumed by the pipeline user to operate on this stage. "
+    "Provide it only if you want to user your own role, otherwise, the command will create one",
+    required=False,
+)
+@click.option(
+    "--cloudformation-execution-role",
+    help="The ARN of an IAM role to be assumed by the CloudFormation service while deploying the application's stack. "
+    "Provide it only if you want to user your own role, otherwise, the command will create one.",
+    required=False,
+)
+@click.option(
+    "--artifacts-bucket",
+    help="The ARN of an S3 bucket to hold the AWS SAM build artifacts. "
+    "Provide it only if you want to user your own S3 bucket, otherwise, the command will create one.",
+    required=False,
+)
+@click.option(
+    "--create-ecr-repo/--no-create-ecr-repo",
+    is_flag=True,
+    default=False,
+    help="If set to true and no ECR repository is provided, this command will create an ECR repository to hold the"
+    " container images of Lambda functions having an Image package type.",
+)
+@click.option(
+    "--ecr-repo",
+    help="The ARN of an ECR repository to hold the containers images of Lambda functions of Image package type. "
+    "If provided, the --create-ecr-repo argument is ignored. If not provided and --create-ecr-repo is set to true, "
+    "the command will create one.",
+    required=False,
+)
+@click.option(
+    "--pipeline-ip-range",
+    help="If provided, all requests coming from outside of the given range are denied. Example: 10.24.34.0/24",
+    required=False,
+)
+@click.option(
+    "--confirm-changeset/--no-confirm-changeset",
+    default=True,
+    is_flag=True,
+    help="Prompt to confirm if the resources is to be deployed by SAM CLI.",
+)
+@common_options
+@aws_creds_options
+@pass_context
+@track_command
+@check_newer_version
+@print_cmdline_args
+def cli(
+    ctx: Any,
+    interactive: bool,
+    stage_name: Optional[str],
+    pipeline_user: Optional[str],
+    pipeline_execution_role: Optional[str],
+    cloudformation_execution_role: Optional[str],
+    artifacts_bucket: Optional[str],
+    create_ecr_repo: bool,
+    ecr_repo: Optional[str],
+    pipeline_ip_range: Optional[str],
+    confirm_changeset: bool,
+    config_file: Optional[str],
+    config_env: Optional[str],
+) -> None:
+    """
+    `sam pipeline bootstrap` command entry point
+    """
+    do_cli(
+        region=ctx.region,
+        profile=ctx.profile,
+        interactive=interactive,
+        stage_name=stage_name,
+        pipeline_user_arn=pipeline_user,
+        pipeline_execution_role_arn=pipeline_execution_role,
+        cloudformation_execution_role_arn=cloudformation_execution_role,
+        artifacts_bucket_arn=artifacts_bucket,
+        create_ecr_repo=create_ecr_repo,
+        ecr_repo_arn=ecr_repo,
+        pipeline_ip_range=pipeline_ip_range,
+        confirm_changeset=confirm_changeset,
+        config_file=config_env,
+        config_env=config_file,
+    )  # pragma: no cover
+
+
+def do_cli(
+    region: Optional[str],
+    profile: Optional[str],
+    interactive: bool,
+    stage_name: Optional[str],
+    pipeline_user_arn: Optional[str],
+    pipeline_execution_role_arn: Optional[str],
+    cloudformation_execution_role_arn: Optional[str],
+    artifacts_bucket_arn: Optional[str],
+    create_ecr_repo: bool,
+    ecr_repo_arn: Optional[str],
+    pipeline_ip_range: Optional[str],
+    confirm_changeset: bool,
+    config_file: Optional[str],
+    config_env: Optional[str],
+) -> None:
+    """
+    implementation of `sam pipeline bootstrap` command
+    """
+    if not pipeline_user_arn:
+        pipeline_user_arn = _load_saved_pipeline_user_arn()
+
+    if interactive:
+        guided_context = GuidedContext(
+            stage_name=stage_name,
+            pipeline_user_arn=pipeline_user_arn,
+            pipeline_execution_role_arn=pipeline_execution_role_arn,
+            cloudformation_execution_role_arn=cloudformation_execution_role_arn,
+            artifacts_bucket_arn=artifacts_bucket_arn,
+            create_ecr_repo=create_ecr_repo,
+            ecr_repo_arn=ecr_repo_arn,
+            pipeline_ip_range=pipeline_ip_range,
+        )
+        guided_context.run()
+        stage_name = guided_context.stage_name
+        pipeline_user_arn = guided_context.pipeline_user_arn
+        pipeline_execution_role_arn = guided_context.pipeline_execution_role_arn
+        pipeline_ip_range = guided_context.pipeline_ip_range
+        cloudformation_execution_role_arn = guided_context.cloudformation_execution_role_arn
+        artifacts_bucket_arn = guided_context.artifacts_bucket_arn
+        create_ecr_repo = guided_context.create_ecr_repo
+        ecr_repo_arn = guided_context.ecr_repo_arn
+
+    if not stage_name:
+        raise click.UsageError("Missing required parameter '--stage-name'")
+
+    stage: Stage = Stage(
+        name=stage_name,
+        aws_profile=profile,
+        aws_region=region,
+        pipeline_user_arn=pipeline_user_arn,
+        pipeline_execution_role_arn=pipeline_execution_role_arn,
+        pipeline_ip_range=pipeline_ip_range,
+        cloudformation_execution_role_arn=cloudformation_execution_role_arn,
+        artifacts_bucket_arn=artifacts_bucket_arn,
+        create_ecr_repo=create_ecr_repo,
+        ecr_repo_arn=ecr_repo_arn,
+    )
+
+    bootstrapped: bool = stage.bootstrap(confirm_changeset=confirm_changeset)
+
+    if bootstrapped:
+        stage.print_resources_summary()
+
+        stage.save_config_safe(
+            config_dir=PIPELINE_CONFIG_DIR, filename=PIPELINE_CONFIG_FILENAME, cmd_names=_get_command_names()
+        )
+
+
+def _load_saved_pipeline_user_arn() -> Optional[str]:
+    samconfig: SamConfig = SamConfig(config_dir=PIPELINE_CONFIG_DIR, filename=PIPELINE_CONFIG_FILENAME)
+    if not samconfig.exists():
+        return None
+    config: Dict[str, str] = samconfig.get_all(cmd_names=_get_command_names(), section="parameters")
+    return config.get("pipeline_user")
+
+
+def _get_command_names() -> List[str]:
+    ctx = click.get_current_context()
+    return get_cmd_names(ctx.info_name, ctx)  # ["pipeline", "bootstrap"]
diff --git a/samcli/commands/pipeline/bootstrap/guided_context.py b/samcli/commands/pipeline/bootstrap/guided_context.py
@@ -0,0 +1,96 @@
+"""
+An interactive flow that prompt the user for required information to bootstrap the AWS account of a pipeline stage
+with the required infrastructure
+"""
+from typing import Optional
+
+import click
+
+
+class GuidedContext:
+    def __init__(
+        self,
+        stage_name: Optional[str] = None,
+        pipeline_user_arn: Optional[str] = None,
+        pipeline_execution_role_arn: Optional[str] = None,
+        cloudformation_execution_role_arn: Optional[str] = None,
+        artifacts_bucket_arn: Optional[str] = None,
+        create_ecr_repo: bool = False,
+        ecr_repo_arn: Optional[str] = None,
+        pipeline_ip_range: Optional[str] = None,
+    ) -> None:
+        self.stage_name = stage_name
+        self.pipeline_user_arn = pipeline_user_arn
+        self.pipeline_execution_role_arn = pipeline_execution_role_arn
+        self.cloudformation_execution_role_arn = cloudformation_execution_role_arn
+        self.artifacts_bucket_arn = artifacts_bucket_arn
+        self.create_ecr_repo = create_ecr_repo
+        self.ecr_repo_arn = ecr_repo_arn
+        self.pipeline_ip_range = pipeline_ip_range
+
+    def run(self) -> None:
+        """
+        Runs an interactive questionnaire to prompt the user for the ARNs of the AWS resources(infrastructure) required
+        for the pipeline to work. Users can provide all, none or some resources' ARNs and leave the remaining empty
+        and it will be created by the bootstrap command
+        """
+        if not self.stage_name:
+            self.stage_name = click.prompt("Stage Name", type=click.STRING)
+
+        if not self.pipeline_user_arn:
+            click.echo(
+                "\nThere must be exactly one pipeline user across all of the pipeline stages. "
+                "If you have ran this command before to bootstrap a previous pipeline stage, please "
+                "provide the ARN of the created pipeline user, otherwise, we will create a new user for you. "
+                "Please make sure to store the credentials safely with the CI/CD provider."
+            )
+            self.pipeline_user_arn = click.prompt(
+                "Pipeline user [leave blank to create one]", default="", type=click.STRING
+            )
+
+        if not self.pipeline_execution_role_arn:
+            self.pipeline_execution_role_arn = click.prompt(
+                "\nPipeline execution role (an IAM role assumed by the pipeline user to operate on this stage) "
+                "[leave blank to create one]",
+                default="",
+                type=click.STRING,
+            )
+
+        if not self.cloudformation_execution_role_arn:
+            self.cloudformation_execution_role_arn = click.prompt(
+                "\nCloudFormation execution role (an IAM role assumed by CloudFormation to deploy "
+                "the application's stack) [leave blank to create one]",
+                default="",
+                type=click.STRING,
+            )
+
+        if not self.artifacts_bucket_arn:
+            self.artifacts_bucket_arn = click.prompt(
+                "\nArtifacts bucket (S3 bucket to hold the AWS SAM build artifacts) [leave blank to create one]",
+                default="",
+                type=click.STRING,
+            )
+        if not self.ecr_repo_arn:
+            click.echo(
+                "\nIf your SAM template includes (or going to include) Lambda functions of Image package type, "
+                "then an ECR repository is required. Should we create one?"
+            )
+            click.echo("\t1 - No, My SAM template won't include Lambda functions of Image package type")
+            click.echo("\t2 - Yes, I need help creating one")
+            click.echo("\t3 - I already have an ECR repository")
+            choice = click.prompt(text="Choice", show_choices=False, type=click.Choice(["1", "2", "3"]))
+            if choice == "1":
+                self.create_ecr_repo = False
+            elif choice == "2":
+                self.create_ecr_repo = True
+            else:  # choice == "3"
+                self.create_ecr_repo = False
+                self.ecr_repo_arn = click.prompt("ECR repo", type=click.STRING)
+
+        if not self.pipeline_ip_range:
+            click.echo("\nWe can deny requests not coming from a recognized IP address range.")
+            self.pipeline_ip_range = click.prompt(
+                "Pipeline IP address range (using CIDR notation) [leave blank if you don't know]",
+                default="",
+                type=click.STRING,
+            )
diff --git a/samcli/commands/pipeline/pipeline.py b/samcli/commands/pipeline/pipeline.py
@@ -0,0 +1,19 @@
+"""
+Command group for "pipeline" suite for commands. It provides common CLI arguments, template parsing capabilities,
+setting up stdin/stdout etc
+"""
+
+import click
+
+from .bootstrap.cli import cli as bootstrap_cli
+
+
+@click.group()
+def cli() -> None:
+    """
+    Manage the continuous delivery of the application
+    """
+
+
+# Add individual commands under this group
+cli.add_command(bootstrap_cli)