From 06f2309f7f1c7aee8f4211d0e6490d39e28ec0d1 Mon Sep 17 00:00:00 2001 From: sasuolander Date: Tue, 3 May 2022 10:34:27 +0300 Subject: [PATCH 01/37] DROTH-3228 add missing webhook --- aws/cloud-formation/cicd/qa/cicd-qa.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/aws/cloud-formation/cicd/qa/cicd-qa.yaml b/aws/cloud-formation/cicd/qa/cicd-qa.yaml index 12ecc7fd53..7f88ea06ad 100644 --- a/aws/cloud-formation/cicd/qa/cicd-qa.yaml +++ b/aws/cloud-formation/cicd/qa/cicd-qa.yaml @@ -69,6 +69,21 @@ Resources: Bool: 'aws:SecureTransport': false + AppPipelineWebhookQA: + Type: 'AWS::CodePipeline::Webhook' + Properties: + Authentication: GITHUB_HMAC + AuthenticationConfiguration: + SecretToken: !Ref GitHubWebhookSecret + Filters: + - JsonPath: $.ref + MatchEquals: 'refs/heads/{Branch}' + TargetPipeline: !Ref AppPipeline + TargetAction: SourceAction + Name: AppPipelineWebhookQA + TargetPipelineVersion: !GetAtt AppPipeline.Version + RegisterWithThirdParty: true + AppPipeline: Type: 'AWS::CodePipeline::Pipeline' Properties: From 471bb2b078c6036fb0ed497aa92cac96e34b9b90 Mon Sep 17 00:00:00 2001 From: "antti.ahopelto" Date: Fri, 6 May 2022 09:53:14 +0300 Subject: [PATCH 02/37] DROTH-3214 Batch system divided for prod and dev account. Added CloudFormation for lambda and it's pipeline --- .../batchSystem/batch-parameter.json | 26 -- .../batchSystem/batchLambda/batchLambda.yaml | 45 ++ .../cicd/batch-lambda-pipeline-parameter.json | 18 + .../batchLambda/cicd/batchLambdaBuildSpec.yml | 21 + .../batchLambda/cicd/batchLambdaPipeline.yaml | 205 +++++++++ .../cicd/prodBatchLambdaDeploymentBucket.yaml | 31 ++ .../dev-batch-lambda-parameter.json | 10 + .../prod-batch-lambda-parameter.json | 10 + .../batchSystem/batchSystem.yaml | 408 ++++-------------- .../dev-batch-system-parameter.json | 34 ++ .../prod-batch-system-parameter.json | 34 ++ .../qa-batch-system-parameter.json | 34 ++ 12 files changed, 531 insertions(+), 345 deletions(-) delete mode 100644 aws/cloud-formation/batchSystem/batch-parameter.json create mode 100644 aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml create mode 100644 aws/cloud-formation/batchSystem/batchLambda/cicd/batch-lambda-pipeline-parameter.json create mode 100644 aws/cloud-formation/batchSystem/batchLambda/cicd/batchLambdaBuildSpec.yml create mode 100644 aws/cloud-formation/batchSystem/batchLambda/cicd/batchLambdaPipeline.yaml create mode 100644 aws/cloud-formation/batchSystem/batchLambda/cicd/prodBatchLambdaDeploymentBucket.yaml create mode 100644 aws/cloud-formation/batchSystem/batchLambda/dev-batch-lambda-parameter.json create mode 100644 aws/cloud-formation/batchSystem/batchLambda/prod-batch-lambda-parameter.json create mode 100644 aws/cloud-formation/batchSystem/dev-batch-system-parameter.json create mode 100644 aws/cloud-formation/batchSystem/prod-batch-system-parameter.json create mode 100644 aws/cloud-formation/batchSystem/qa-batch-system-parameter.json diff --git a/aws/cloud-formation/batchSystem/batch-parameter.json b/aws/cloud-formation/batchSystem/batch-parameter.json deleted file mode 100644 index 7e67dbbccd..0000000000 --- a/aws/cloud-formation/batchSystem/batch-parameter.json +++ /dev/null @@ -1,26 +0,0 @@ -[ - { - "ParameterKey": "DEVNetworkStackName", - "ParameterValue": "Digiroad-OTH-kehitys" - }, - { - "ParameterKey": "QANetworkStackName", - "ParameterValue": "DigiroadTest" - }, - { - "ParameterKey": "DEVVpcIDOfSystem", - "ParameterValue": "vpc-0f430b7fedef04ba3" - }, - { - "ParameterKey": "QAVpcIDOfSystem", - "ParameterValue": "vpc-0b4e33ad8202e91e4" - }, - { - "ParameterKey": "SNSTopicName", - "ParameterValue": "BatchFailTopic" - }, - { - "ParameterKey": "BatchLambdaArn", - "ParameterValue": "arn:aws:lambda:eu-west-1:475079312496:function:Batch-Add-Jobs-To-Queue" - } -] \ No newline at end of file diff --git a/aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml b/aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml new file mode 100644 index 0000000000..7e1331abed --- /dev/null +++ b/aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml @@ -0,0 +1,45 @@ +AWSTemplateFormatVersion: "2010-09-09" +Parameters: + BucketName: + Description: Name of S3 bucket for lambda code + Type: String + S3ObjectKey: + Description: Key of lambda deployment object + Type: String + +Resources: + BatchLambda: + Type: AWS::Lambda::Function + Properties: + Code: + S3Bucket: !ImportValue 'BatchLambda-deployment-bucket' + S3Key: batch-lambda-deployment + FunctionName: "Batch-Add-Jobs-To-Queue" + Handler: AddJobToQueue.handler + Role: !GetAtt BatchLambdaRole.Arn + Runtime: nodejs14.x + + BatchLambdaRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - 'sts:AssumeRole' + - 'logs:CreateLogStream' + - 'logs:PutLogEvents' + - 'batch:DescribeJobs' + - 'batch:SubmitJob' + - 'batch:ListJobs' + +Outputs: + BatchLambdaOutput: + Description: Arn of lambda for adding jobs to queue + Value: !Ref BatchLambda + Export: + Name: !Sub "${AWS::StackName}-BatchLambdaID" \ No newline at end of file diff --git a/aws/cloud-formation/batchSystem/batchLambda/cicd/batch-lambda-pipeline-parameter.json b/aws/cloud-formation/batchSystem/batchLambda/cicd/batch-lambda-pipeline-parameter.json new file mode 100644 index 0000000000..0db6bd197f --- /dev/null +++ b/aws/cloud-formation/batchSystem/batchLambda/cicd/batch-lambda-pipeline-parameter.json @@ -0,0 +1,18 @@ +[ + { + "ParameterKey": "DevBucketName", + "ParameterValue": "dev-batch-lambda-deployment-bucket" + }, + { + "ParameterKey": "ProdBucketName", + "ParameterValue": "prod-batch-lambda-deployment-bucket" + }, + { + "ParameterKey": "CodeCommitRepositoryName", + "ParameterValue": "Batch-Lambda" + }, + { + "ParameterKey": "CodeCommitBranch", + "ParameterValue": "master" + } +] \ No newline at end of file diff --git a/aws/cloud-formation/batchSystem/batchLambda/cicd/batchLambdaBuildSpec.yml b/aws/cloud-formation/batchSystem/batchLambda/cicd/batchLambdaBuildSpec.yml new file mode 100644 index 0000000000..6d110385c2 --- /dev/null +++ b/aws/cloud-formation/batchSystem/batchLambda/cicd/batchLambdaBuildSpec.yml @@ -0,0 +1,21 @@ +version: 0.2 +phases: + install: + runtime-versions: + nodejs: 14.x + commands: + - mkdir -p ./lib + - npm install --prefix ./lib aws-sdk + build: + commands: + - cd lib + - zip -r9 ../deployment_package.zip . + - cd .. + - zip -g deployment_package.zip AddJobToQueue.js + - echo Pushing deployment_package.zip to Prod and Dev S3 buckets + - aws s3api put-object --bucket $DEV_BUCKET --key batch-lambda-deployment --body deployment_package.zip + - aws s3api put-object --bucket $PROD_BUCKET --key batch-lambda-deployment --body deployment_package.zip --acl bucket-owner-full-control + post_build: + commands: + - echo Updating Dev account lambda code + - aws lambda update-function-code --function-name Batch-Add-Jobs-To-Queue --zip-file fileb://deployment_package.zip \ No newline at end of file diff --git a/aws/cloud-formation/batchSystem/batchLambda/cicd/batchLambdaPipeline.yaml b/aws/cloud-formation/batchSystem/batchLambda/cicd/batchLambdaPipeline.yaml new file mode 100644 index 0000000000..2ed8fee710 --- /dev/null +++ b/aws/cloud-formation/batchSystem/batchLambda/cicd/batchLambdaPipeline.yaml @@ -0,0 +1,205 @@ +AWSTemplateFormatVersion: "2010-09-09" +Parameters: + DevBucketName: + Description: Name of dev account S3 bucket for lambda deployment + Type: String + ProdBucketName: + Description: Name of prod account S3 bucket for lambda deployment + Type: String + CodeCommitRepositoryName: + Description: Name of CodeCommit repo for lambda source code + Type: String + CodeCommitBranch: + Description: Branch to use for lambda source code + Type: String + +Resources: + LambdaDeploymentBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Ref DevBucketName + + BatchLambdaPipeline: + Type: AWS::CodePipeline::Pipeline + Properties: + Name: 'batch-lambda-pipeline' + RoleArn: !GetAtt PipelineRole.Arn + RestartExecutionOnUpdate: true + Stages: + - Name: Source + Actions: + Name: SourceAction + ActionTypeID: + Category: Source + Owner: AWS + Provider: CodeCommit + Version: 1 + OutputArtifacts: + - Name: SourceOutput + Configuration: + RepositoryName: !Ref CodeCommitRepositoryName + BranchName: !Ref CodeCommitBranch + PollForSourceChanges: false + RunOrder: 1 + - Name: Build + Actions: + Name: BuildAction + InputArtifacts: + - Name: SourceOutput + ActionTypeId: + Category: Build + Owner: AWS + Version: 1 + Provider: CodeBuild + OutputArtifacts: + - Name: Built + Configuration: + ProjectName: !Ref CodeBuild + RunOrder: 1 + + PipelineRole: + Type: 'AWS::IAM::Role' + Properties: + RoleName: 'batch-lambda-pipeline-role' + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - codepipeline.amazonaws.com + Action: 'sts:AssumeRole' + Path: / + Policies: + - PolicyName: 'batch-lambda-pipeline-policy' + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - 'codebuild:BatchGetBuilds' + - 'codebuild:StartBuild' + Resource: '*' + - Effect: Allow + Action: + - 'lambda:*' + - 'codebuild:*' + - 's3:*' + Resource: '*' + + StartPipeLineEventRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - events.amazonaws.com + Action: sts:AssumeRole + Path: / + Policies: + - PolicyName: cwe-batch-lambda-pipeline-execution + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: codepipeline:StartPipelineExecution + Resource: !Ref BatchLambdaPipeline + + StartPipelineRule: + Type: AWS::Events::Rule + Properties: + EventPattern: + source: + - aws.codecommit + detail-type: + - 'CodeCommit Repository State Change' + resources: + - arn:aws:codecommit:eu-west-1:475079312496:Batch-Lambda + detail: + event: + - referenceCreated + - referenceUpdated + referenceType: + - branch + referenceName: + - master + Targets: + - Arn: !GetAtt BatchLambdaPipeline.Arn + RoleArn: !GetAtt StartPipeLineEventRole.Arn + Id: batch-lambda-pipeline + + CodeBuild: + Type: AWS::CodeBuild::Project + Properties: + Name: 'batch-lambda-codebuild' + Description: 'CodeBuild for batch-lambda' + ServiceRole: !Ref CodeBuildRole + Environment: + ComputeType: BUILD_GENERAL1_SMALL + Image: aws/codebuild/standard:5.0 + Type: LINUX_CONTAINER + PrivilegedMode: true + EnvironmentVariables: + - Name: DEV_BUCKET + Value: !Ref DevBucketName + - Name: PROD_BUCKET + Value: !Ref ProdBucketName + Artifacts: + Name: 'batch-lambda-build-artifact' + Type: CODEPIPELINE + Source: + Type: CODEPIPELINE + BuildSpec: 'batchLambdaBuildSpec.yml' + + CodeBuildRole: + Type: AWS::IAM::Role + Properties: + Path: / + RoleName: "batch-lambda-codebuild-role" + AssumeRolePolicyDocument: + Statement: + - Effect: "Allow" + Principal: + Service: "codebuild.amazonaws.com" + Action: + - "sts:AssumeRole" + Policies: + - PolicyName: "batch-lambda-codebuild-policy" + PolicyDocument: + Statement: + - Effect: "Allow" + Resource: "*" + Action: + - "ecr:GetAuthorizationToken" + - "ecr:BatchCheckLayerAvailability" + - "ecr:GetDownloadUrlForLayer" + - "ecr:GetRepositoryPolicy" + - "ecr:DescribeRepositories" + - "ecr:ListImages" + - "ecr:DescribeImages" + - "ecr:BatchGetImage" + - "ecr:GetLifecyclePolicy" + - "ecr:GetLifecyclePolicyPreview" + - "ecr:ListTagsForResource" + - "ecr:DescribeImageScanFindings" + - "ecr:InitiateLayerUpload" + - "ecr:UploadLayerPart" + - "ecr:CompleteLayerUpload" + - "ecr:PutImage" + - "ssm:GetParameters" + - Effect: "Allow" + Resource: "*" + Action: + - "logs:CreateLogGroup" + - "logs:CreateLogStream" + - "logs:PutLogEvents" + - Effect: "Allow" + Resource: + - "*" + Action: + - "s3:GetObject" + - "s3:GetObjectVersion" + - "s3:PutObject" \ No newline at end of file diff --git a/aws/cloud-formation/batchSystem/batchLambda/cicd/prodBatchLambdaDeploymentBucket.yaml b/aws/cloud-formation/batchSystem/batchLambda/cicd/prodBatchLambdaDeploymentBucket.yaml new file mode 100644 index 0000000000..72630d0409 --- /dev/null +++ b/aws/cloud-formation/batchSystem/batchLambda/cicd/prodBatchLambdaDeploymentBucket.yaml @@ -0,0 +1,31 @@ +AWSTemplateFormatVersion: "2010-09-09" + +Resources: + LambdaDeploymentBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: 'prod-batch-lambda-deployment-bucket' + AccessControl: 'private' + + DeploymentBucketPolicy: + Type: AWS::S3::BucketPolicy + Properties: + Bucket: !Ref LambdaDeploymentBucket + PolicyDocument: + Statement: + Sid: "PolicyForAllowUploadWithACL" + Effect: Allow + Principal: + AWS: '475079312496' + Action: "s3:PutObject" + Resource: "arn:aws:s3:::prod-batch-lambda-deployment-bucket/*" + Condition: { + "StringEquals": {"s3:x-amz-acl":"bucket-owner-full-control"} + } + +Outputs: + BucketOutput: + Description: 'Name of production batch lambda S3 deployment bucket' + Value: 'prod-batch-lambda-deployment-bucket' + Export: + Name: 'BatchLambda-deployment-bucket' \ No newline at end of file diff --git a/aws/cloud-formation/batchSystem/batchLambda/dev-batch-lambda-parameter.json b/aws/cloud-formation/batchSystem/batchLambda/dev-batch-lambda-parameter.json new file mode 100644 index 0000000000..bb826bd725 --- /dev/null +++ b/aws/cloud-formation/batchSystem/batchLambda/dev-batch-lambda-parameter.json @@ -0,0 +1,10 @@ +[ + { + "ParameterKey": "BucketName", + "ParameterValue": "dev-batch-lambda-deployment-bucket" + }, + { + "ParameterKey": "S3ObjectKey", + "ParameterValue": "batch-lambda-deployment" + } +] \ No newline at end of file diff --git a/aws/cloud-formation/batchSystem/batchLambda/prod-batch-lambda-parameter.json b/aws/cloud-formation/batchSystem/batchLambda/prod-batch-lambda-parameter.json new file mode 100644 index 0000000000..bb826bd725 --- /dev/null +++ b/aws/cloud-formation/batchSystem/batchLambda/prod-batch-lambda-parameter.json @@ -0,0 +1,10 @@ +[ + { + "ParameterKey": "BucketName", + "ParameterValue": "dev-batch-lambda-deployment-bucket" + }, + { + "ParameterKey": "S3ObjectKey", + "ParameterValue": "batch-lambda-deployment" + } +] \ No newline at end of file diff --git a/aws/cloud-formation/batchSystem/batchSystem.yaml b/aws/cloud-formation/batchSystem/batchSystem.yaml index 0e0e583d0b..a7ec2067fc 100644 --- a/aws/cloud-formation/batchSystem/batchSystem.yaml +++ b/aws/cloud-formation/batchSystem/batchSystem.yaml @@ -1,161 +1,77 @@ AWSTemplateFormatVersion: "2010-09-09" Parameters: - DEVNetworkStackName: - Description: Name of DEV network stack + NetworkStackName: + Description: Name of network stack Type: String - QANetworkStackName: - Description: Name of QA network stack - Type: String - DEVVpcIDOfSystem: - Type: AWS::EC2::VPC::Id - Description: DEV VPC of your system - QAVpcIDOfSystem: + VpcIDOfSystem: Type: AWS::EC2::VPC::Id - Description: QA VPC of your system + Description: VPC of your system SNSTopicName: Type: String Description: Name of SNS topic - BatchLambdaArn: + EnvironmentName: + Type: String + Description: Name of used environment + JobDefinitionName: + Type: String + Description: Name of Job Definition that Lambda uses to submit job + BatchLambdaStackName: + Type: String + Description: Stack name of batch lambda, used for ImportValue + Subnet1: + Type: String + Description: Subnet1 Id for ImportValue + Subnet2: Type: String - Description: ARN of lambda function which adds jobs to queue + Description: Subnet2 Id for ImportValue Resources: - DEVJobQueueAdHoc: + JobQueueAdHoc: Type: AWS::Batch::JobQueue Properties: - JobQueueName: "DEV-adhoc" + JobQueueName: !Join [ '-', [ !Ref EnvironmentName, 'adhoc']] Priority: 1 ComputeEnvironmentOrder: - Order: 1 ComputeEnvironment: - Ref: DEVComputeEnvironment - - QAJobQueueAdHoc: - Type: AWS::Batch::JobQueue - Properties: - JobQueueName: "QA-adhoc" - Priority: 1 - ComputeEnvironmentOrder: - - Order: 1 - ComputeEnvironment: - Ref: QAComputeEnviroment - - ProdJobQueueAdHoc: - Type: AWS::Batch::JobQueue - Properties: - JobQueueName: "Prod-adhoc" - Priority: 1 - ComputeEnvironmentOrder: - - Order: 1 - ComputeEnvironment: - Ref: ProdComputeEnviroment + Ref: ComputeEnvironment - DEVJobQueue: + JobQueue: Type: AWS::Batch::JobQueue Properties: - JobQueueName: "DEV-JobQueue" + JobQueueName: !Join [ '-', [ !Ref EnvironmentName, 'jobQueue']] Priority: 1 ComputeEnvironmentOrder: - Order: 1 ComputeEnvironment: - Ref: DEVComputeEnvironment - - QAJobQueue: - Type: AWS::Batch::JobQueue - Properties: - JobQueueName: "QA-JobQueue" - Priority: 1 - ComputeEnvironmentOrder: - - Order: 1 - ComputeEnvironment: - Ref: QAComputeEnviroment - - ProdJobQueue: - Type: AWS::Batch::JobQueue - Properties: - JobQueueName: "Prod-JobQueue" - Priority: 1 - ComputeEnvironmentOrder: - - Order: 1 - ComputeEnvironment: - Ref: ProdComputeEnviroment - - DEVComputeEnvironment: - Type: AWS::Batch::ComputeEnvironment - Properties: - Type: MANAGED - ComputeEnvironmentName: "DEV-BatchCompute" - ComputeResources: - Type: FARGATE - MaxvCpus: 32 - Subnets: - - Fn::ImportValue: - !Join [ '-', [ !Ref DEVNetworkStackName, 'Subnet1-Id' ] ] - - Fn::ImportValue: - !Join [ '-', [ !Ref DEVNetworkStackName, 'Subnet2-Id' ] ] - SecurityGroupIds: - - !Ref DEVBatchSecurityGroup - ServiceRole: - Ref: BatchServiceRole + Ref: ComputeEnvironment - DEVBatchSecurityGroup: - Type: AWS::EC2::SecurityGroup - Properties: - GroupDescription: Allaw inbound to port 80 and 5432, Outbound to everything - GroupName: DEV-BatchSecurityGroup - VpcId: !Ref DEVVpcIDOfSystem - - QAComputeEnviroment: + ComputeEnvironment: Type: AWS::Batch::ComputeEnvironment Properties: Type: MANAGED - ComputeEnvironmentName: "QA-BatchCompute" + ComputeEnvironmentName: !Join [ '-', [ !Ref EnvironmentName, 'BatchCompute']] ComputeResources: Type: FARGATE MaxvCpus: 32 Subnets: - Fn::ImportValue: - !Join [ '-', [ !Ref QANetworkStackName, 'Subnet1-Id' ] ] + !Ref Subnet1 - Fn::ImportValue: - !Join [ '-', [ !Ref QANetworkStackName, 'Subnet2-Id' ] ] + !Ref Subnet2 SecurityGroupIds: - - !Ref QABatchSecurityGroup + - !Ref BatchSecurityGroup ServiceRole: Ref: BatchServiceRole - QABatchSecurityGroup: + BatchSecurityGroup: Type: AWS::EC2::SecurityGroup Properties: - GroupDescription: Allaw inbound to port 80 and 5432, Outbound to everything - GroupName: QA-BatchSecurityGroup - VpcId: !Ref QAVpcIDOfSystem - - ProdComputeEnviroment: - Type: AWS::Batch::ComputeEnvironment - Properties: - Type: MANAGED - ComputeEnvironmentName: "Prod-BatchCompute" - ComputeResources: - Type: FARGATE - MaxvCpus: 32 - Subnets: - - Fn::ImportValue: - !Join [ '-', [ !Ref DEVNetworkStackName, 'Subnet1-Id' ] ] - - Fn::ImportValue: - !Join [ '-', [ !Ref DEVNetworkStackName, 'Subnet2-Id' ] ] - SecurityGroupIds: - - !Ref ProdBatchSecurityGroup - ServiceRole: - Ref: BatchServiceRole + GroupDescription: Allow inbound to port 80 and 5432, Outbound to everything + GroupName: !Join [ '-', [ !Ref EnvironmentName, 'batchSecurityGroup']] + VpcId: !Ref VpcIDOfSystem - ProdBatchSecurityGroup: - Type: AWS::EC2::SecurityGroup - Properties: - GroupDescription: Allaw inbound to port 80 and 5432, Outbound to everything - GroupName: Prod-BatchSecurityGroup - VpcId: !Ref DEVVpcIDOfSystem - BatchTaskRole: Type: AWS::IAM::Role Properties: @@ -167,7 +83,7 @@ Resources: Action: ['sts:AssumeRole'] Path: / Policies: - - PolicyName: AmazonBatchECSTaskExecutionRolePolicy + - PolicyName: !Join [ '-', [ !Ref EnvironmentName, 'AmazonBatchECSTaskExecutionRolePolicy']] PolicyDocument: Statement: - Effect: Allow @@ -231,253 +147,107 @@ Resources: queue: "$.detail.jobQueue" reason: "$.detail.statusReason" - #Prod Batch Events - ProdRunAnnualBatch: + #Batch Events + RunAnnualBatch: Type: AWS::Events::Rule Properties: - Description: "Run annual Prod batches" + Name: !Join [ '-', [ !Ref EnvironmentName, 'BatchAnnualEventRule' ]] + Description: "Run annual batches" ScheduleExpression: "cron(30 22 L 12 ? *)" State: "ENABLED" Targets: - - Arn: !Ref BatchLambdaArn - Input: "{\"jobName\": \"Prod\", \"jobDefinition\": \"ProdBatchJobDefinition\", \"type\":\"annual\"}" - Id: "ProdAnnualLambda" - + Arn: + - !ImportValue + !Sub: '${BatchLambdaStackName}-BatchLambdaID' + Input: !Join [ '', [ '{ \"jobName\": ', '\"', !Ref EnvironmentName, '\", ', '\"jobDefinition\": ', '\"', !Ref JobDefinitionName, '\", ', '\"type\": \"annual\" }' ]] + Id: !Join [ '-' [!Ref EnvironmentName, 'AnnualLambda']] - ProdRunMonthlyBatch: + RunMonthlyBatch: Type: AWS::Events::Rule Properties: - Description: "Run monthly Prod batches" + Name: !Join [ '-', [ !Ref EnvironmentName, 'BatchMonthlyEventRule' ]] + Description: "Run monthly batches" ScheduleExpression: "cron(30 22 14 * ? *)" State: "ENABLED" Targets: - - Arn: !Ref BatchLambdaArn - Input: "{\"jobName\": \"Prod\", \"jobDefinition\": \"ProdBatchJobDefinition\", \"type\":\"monthly\"}" - Id: "ProdMonthlyLambda" + Arn: + - !ImportValue + !Sub: '${BatchLambdaStackName}-BatchLambdaID' + Input: !Join [ '', [ '{ \"jobName\": ', '\"', !Ref EnvironmentName, '\", ', '\"jobDefinition\": ', '\"', !Ref JobDefinitionName, '\", ', '\"type\": \"monthly\" }' ]] + Id: !Join [ '-' [!Ref EnvironmentName, 'MonthlyLambda']] - ProdRunWeeklyBatch: + RunWeeklyBatch: Type: AWS::Events::Rule Properties: - Description: "Run weekly Prod batches" + Name: !Join [ '-', [ !Ref EnvironmentName, 'BatchWeeklyEventRule' ]] + Description: "Run weekly batches" ScheduleExpression: "cron(0 22 ? * 6 *)" State: "ENABLED" Targets: - - Arn: !Ref BatchLambdaArn - Input: "{\"jobName\": \"Prod\", \"jobDefinition\": \"ProdBatchJobDefinition\", \"type\":\"weekly\"}" - Id: "ProdWeeklyLambda" + Arn: + - !ImportValue + !Sub: '${BatchLambdaStackName}-BatchLambdaID' + Input: !Join [ '', [ '{ \"jobName\": ', '\"', !Ref EnvironmentName, '\", ', '\"jobDefinition\": ', '\"', !Ref JobDefinitionName, '\", ', '\"type\": \"weekly\" }' ]] + Id: !Join [ '-' [!Ref EnvironmentName, 'WeeklyLambda']] - ProdRunDailyBatch: + RunDailyBatch: Type: AWS::Events::Rule Properties: - Description: "Run daily Prod batches" + Name: !Join [ '-', [ !Ref EnvironmentName, 'BatchDailyEventRule' ]] + Description: "Run daily batches" ScheduleExpression: "cron(30 22 ? * 1-5 *)" State: "ENABLED" Targets: - - Arn: !Ref BatchLambdaArn - Input: "{\"jobName\": \"Prod\", \"jobDefinition\": \"ProdBatchJobDefinition\", \"type\":\"daily\"}" - Id: "ProdDailyLambda" - - #QA Batch Events - QARunAnnualBatch: - Type: AWS::Events::Rule - Properties: - Description: "Run annual QA batches" - ScheduleExpression: "cron(30 22 L 12 ? *)" - State: "ENABLED" - Targets: - - - Arn: !Ref BatchLambdaArn - Input: "{ \"jobName\": \"QA\", \"jobDefinition\": \"QABatchDefinition\", \"type\": \"annual\" }" - Id: "QAAnnualLambda" - - QARunMonthlyBatch: - Type: AWS::Events::Rule - Properties: - Description: "Run monthly QA batches" - ScheduleExpression: "cron(30 22 14 * ? *)" - State: "ENABLED" - Targets: - - - Arn: !Ref BatchLambdaArn - Input: "{ \"jobName\": \"QA\", \"jobDefinition\": \"QABatchDefinition\", \"type\": \"monthly\" }" - Id: "QAMonthlyLambda" - - - QARunWeeklyBatch: - Type: AWS::Events::Rule - Properties: - Description: "Run weekly Prod batches" - ScheduleExpression: "cron(0 22 ? * 6 *)" - State: "ENABLED" - Targets: - - - Arn: !Ref BatchLambdaArn - Input: "{ \"jobName\": \"QA\", \"jobDefinition\": \"QABatchDefinition\", \"type\": \"weekly\" }" - Id: "QAWeeklyLambda" - - QARunDailyBatch: - Type: AWS::Events::Rule - Properties: - Description: "Run daily QA batches" - ScheduleExpression: "cron(30 22 ? * 1-5 *)" - State: "ENABLED" - Targets: - - - Arn: !Ref BatchLambdaArn - Input: "{ \"jobName\": \"QA\", \"jobDefinition\": \"QABatchDefinition\", \"type\": \"daily\" }" - Id: "QADailyLambda" - - #DEV Batch Events - DEVRunAnnualBatch: - Type: AWS::Events::Rule - Properties: - Description: "Run annual DEV batches" - ScheduleExpression: "cron(30 22 L 12 ? *)" - State: "ENABLED" - Targets: - - - Arn: !Ref BatchLambdaArn - Input: "{ \"jobName\": \"DEV\", \"jobDefinition\": \"DEVBatchDefinition\", \"type\": \"annual\" }" - Id: "DEVAnnualLambda" - - DEVRunMonthlyBatch: - Type: AWS::Events::Rule - Properties: - Description: "Run monthly DEV batches" - ScheduleExpression: "cron(30 22 14 * ? *)" - State: "ENABLED" - Targets: - - - Arn: !Ref BatchLambdaArn - Input: "{ \"jobName\": \"DEV\", \"jobDefinition\": \"DEVBatchDefinition\", \"type\": \"monthly\" }" - Id: "DEVMonthlyLambda" - - DEVRunWeeklyBatch: - Type: AWS::Events::Rule - Properties: - Description: "Run weekly DEV batches" - ScheduleExpression: "cron(0 22 ? * 6 *)" - State: "ENABLED" - Targets: - - - Arn: !Ref BatchLambdaArn - Input: "{ \"jobName\": \"DEV\", \"jobDefinition\": \"DEVBatchDefinition\", \"type\": \"weekly\" }" - Id: "DEVWeeklyLambda" - - DEVRunDailyBatch: - Type: AWS::Events::Rule - Properties: - Description: "Run daily DEV batches" - ScheduleExpression: "cron(30 22 ? * 1-5 *)" - State: "ENABLED" - Targets: - - - Arn: !Ref BatchLambdaArn - Input: "{ \"jobName\": \"DEV\", \"jobDefinition\": \"DEVBatchDefinition\", \"type\": \"daily\" }" - Id: "DEVDailyLambda" - - #Lambda permissions for prod events - ProdAnnualLambdaPermission: - Type: AWS::Lambda::Permission - Properties: - Action: lambda:InvokeFunction - FunctionName: !Ref BatchLambdaArn - Principal: events.amazonaws.com - SourceArn: !GetAtt ProdRunAnnualBatch.Arn - - ProdMonthlyLambdaPermission: - Type: AWS::Lambda::Permission - Properties: - Action: lambda:InvokeFunction - FunctionName: !Ref BatchLambdaArn - Principal: events.amazonaws.com - SourceArn: !GetAtt ProdRunMonthlyBatch.Arn - - ProdWeeklyLambdaPermission: - Type: AWS::Lambda::Permission - Properties: - Action: lambda:InvokeFunction - FunctionName: !Ref BatchLambdaArn - Principal: events.amazonaws.com - SourceArn: !GetAtt ProdRunWeeklyBatch.Arn - - ProdDailyLambdaPermission: - Type: AWS::Lambda::Permission - Properties: - Action: lambda:InvokeFunction - FunctionName: !Ref BatchLambdaArn - Principal: events.amazonaws.com - SourceArn: !GetAtt ProdRunDailyBatch.Arn - - #Lambda permissions for QA events - QAAnnualLambdaPermission: - Type: AWS::Lambda::Permission - Properties: - Action: lambda:InvokeFunction - FunctionName: !Ref BatchLambdaArn - Principal: events.amazonaws.com - SourceArn: !GetAtt QARunAnnualBatch.Arn - - QAMonthlyLambdaPermission: - Type: AWS::Lambda::Permission - Properties: - Action: lambda:InvokeFunction - FunctionName: !Ref BatchLambdaArn - Principal: events.amazonaws.com - SourceArn: !GetAtt QARunMonthlyBatch.Arn - - QAWeeklyLambdaPermission: - Type: AWS::Lambda::Permission - Properties: - Action: lambda:InvokeFunction - FunctionName: !Ref BatchLambdaArn - Principal: events.amazonaws.com - SourceArn: !GetAtt QARunWeeklyBatch.Arn - - QADailyLambdaPermission: - Type: AWS::Lambda::Permission - Properties: - Action: lambda:InvokeFunction - FunctionName: !Ref BatchLambdaArn - Principal: events.amazonaws.com - SourceArn: !GetAtt QARunDailyBatch.Arn - - #Lambda permissions for DEV events - DEVAnnualLambdaPermission: + Arn: + - !ImportValue + !Sub: '${BatchLambdaStackName}-BatchLambdaID' + Input: !Join [ '', [ '{ \"jobName\": ', '\"', !Ref EnvironmentName, '\", ', '\"jobDefinition\": ', '\"', !Ref JobDefinitionName, '\", ', '\"type\": \"daily\" }' ]] + Id: !Join [ '-' [!Ref EnvironmentName, 'DailyLambda']] + + #Lambda permissions for events + AnnualLambdaPermission: Type: AWS::Lambda::Permission Properties: Action: lambda:InvokeFunction - FunctionName: !Ref BatchLambdaArn + FunctionName: + - !ImportValue + !Sub: '${BatchLambdaStackName}-BatchLambdaID' Principal: events.amazonaws.com - SourceArn: !GetAtt DEVRunAnnualBatch.Arn + SourceArn: !GetAtt RunAnnualBatch.Arn - DEVMonthlyLambdaPermission: + MonthlyLambdaPermission: Type: AWS::Lambda::Permission Properties: Action: lambda:InvokeFunction - FunctionName: !Ref BatchLambdaArn + FunctionName: + - !ImportValue + !Sub: '${BatchLambdaStackName}-BatchLambdaID' Principal: events.amazonaws.com - SourceArn: !GetAtt DEVRunMonthlyBatch.Arn + SourceArn: !GetAtt RunMonthlyBatch.Arn - DEVWeeklyLambdaPermission: + WeeklyLambdaPermission: Type: AWS::Lambda::Permission Properties: Action: lambda:InvokeFunction - FunctionName: !Ref BatchLambdaArn + FunctionName: + - !ImportValue + !Sub: '${BatchLambdaStackName}-BatchLambdaID' Principal: events.amazonaws.com - SourceArn: !GetAtt DEVRunWeeklyBatch.Arn + SourceArn: !GetAtt RunWeeklyBatch.Arn - DEVDailyLambdaPermission: + DailyLambdaPermission: Type: AWS::Lambda::Permission Properties: Action: lambda:InvokeFunction - FunctionName: !Ref BatchLambdaArn + FunctionName: + - !ImportValue + !Sub: '${BatchLambdaStackName}-BatchLambdaID' Principal: events.amazonaws.com - SourceArn: !GetAtt DEVRunDailyBatch.Arn + SourceArn: !GetAtt RunDailyBatch.Arn Outputs: BatchTaskRoleARN: diff --git a/aws/cloud-formation/batchSystem/dev-batch-system-parameter.json b/aws/cloud-formation/batchSystem/dev-batch-system-parameter.json new file mode 100644 index 0000000000..935e164ce1 --- /dev/null +++ b/aws/cloud-formation/batchSystem/dev-batch-system-parameter.json @@ -0,0 +1,34 @@ +[ + { + "ParameterKey": "NetworkStackName", + "ParameterValue": "Digiroad-OTH-kehitys" + }, + { + "ParameterKey": "VpcIDOfSystem", + "ParameterValue": "vpc-0f430b7fedef04ba3" + }, + { + "ParameterKey": "SNSTopicName", + "ParameterValue": "BatchFailTopic" + }, + { + "ParameterKey": "EnvironmentName", + "ParameterValue": "DEV" + }, + { + "ParameterKey": "JobDefinitionName", + "ParameterValue": "DEVBatchDefinition" + }, + { + "ParameterKey": "BatchLambdaStackName", + "ParameterValue": "BatchLambdaStack" + }, + { + "ParameterKey": "Subnet1", + "ParameterValue": "Digiroad-OTH-kehitys-Subnet1-Id" + }, + { + "ParameterKey": "Subnet2", + "ParameterValue": "Digiroad-OTH-kehitys-Subnet2-Id" + } +] \ No newline at end of file diff --git a/aws/cloud-formation/batchSystem/prod-batch-system-parameter.json b/aws/cloud-formation/batchSystem/prod-batch-system-parameter.json new file mode 100644 index 0000000000..de67d732eb --- /dev/null +++ b/aws/cloud-formation/batchSystem/prod-batch-system-parameter.json @@ -0,0 +1,34 @@ +[ + { + "ParameterKey": "NetworkStackName", + "ParameterValue": "Digiroad-prod-VPC" + }, + { + "ParameterKey": "VpcIDOfSystem", + "ParameterValue": "vpc-015f75cef3e99c5cc" + }, + { + "ParameterKey": "SNSTopicName", + "ParameterValue": "BatchFailTopic" + }, + { + "ParameterKey": "EnvironmentName", + "ParameterValue": "Prod" + }, + { + "ParameterKey": "JobDefinitionName", + "ParameterValue": "ProdBatchJobDefinition" + }, + { + "ParameterKey": "BatchLambdaStackName", + "ParameterValue": "BatchLambdaStack" + }, + { + "ParameterKey": "Subnet1", + "ParameterValue": "Digiroad-prod-Subnet1-Id" + }, + { + "ParameterKey": "Subnet2", + "ParameterValue": "Digiroad-prod-Subnet2-Id" + } +] \ No newline at end of file diff --git a/aws/cloud-formation/batchSystem/qa-batch-system-parameter.json b/aws/cloud-formation/batchSystem/qa-batch-system-parameter.json new file mode 100644 index 0000000000..5c8a10ac52 --- /dev/null +++ b/aws/cloud-formation/batchSystem/qa-batch-system-parameter.json @@ -0,0 +1,34 @@ +[ + { + "ParameterKey": "NetworkStackName", + "ParameterValue": "DigiroadTest-VPC" + }, + { + "ParameterKey": "VpcIDOfSystem", + "ParameterValue": "vpc-0b4e33ad8202e91e4" + }, + { + "ParameterKey": "SNSTopicName", + "ParameterValue": "BatchFailTopic" + }, + { + "ParameterKey": "EnvironmentName", + "ParameterValue": "QA" + }, + { + "ParameterKey": "JobDefinitionName", + "ParameterValue": "QABatchDefinition" + }, + { + "ParameterKey": "BatchLambdaStackName", + "ParameterValue": "BatchLambdaStack" + }, + { + "ParameterKey": "Subnet1", + "ParameterValue": "DigiroadTest-Subnet1-Id" + }, + { + "ParameterKey": "Subnet2", + "ParameterValue": "DigiroadTest-Subnet2-Id" + } +] \ No newline at end of file From dd60d3dceceb12f43574c951c6263ea334d4643f Mon Sep 17 00:00:00 2001 From: "antti.ahopelto" Date: Fri, 6 May 2022 12:58:18 +0300 Subject: [PATCH 03/37] DROTH-3214 Added instructions for creating stacks --- .../dev-batch-system-parameter.json | 2 +- .../prod-batch-system-parameter.json | 2 +- .../fargateService/prod/README.md | 26 +++++++++++++++++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/aws/cloud-formation/batchSystem/dev-batch-system-parameter.json b/aws/cloud-formation/batchSystem/dev-batch-system-parameter.json index 935e164ce1..87a50225fb 100644 --- a/aws/cloud-formation/batchSystem/dev-batch-system-parameter.json +++ b/aws/cloud-formation/batchSystem/dev-batch-system-parameter.json @@ -21,7 +21,7 @@ }, { "ParameterKey": "BatchLambdaStackName", - "ParameterValue": "BatchLambdaStack" + "ParameterValue": "digiroad-batch-lambda-stack" }, { "ParameterKey": "Subnet1", diff --git a/aws/cloud-formation/batchSystem/prod-batch-system-parameter.json b/aws/cloud-formation/batchSystem/prod-batch-system-parameter.json index de67d732eb..df76969da7 100644 --- a/aws/cloud-formation/batchSystem/prod-batch-system-parameter.json +++ b/aws/cloud-formation/batchSystem/prod-batch-system-parameter.json @@ -21,7 +21,7 @@ }, { "ParameterKey": "BatchLambdaStackName", - "ParameterValue": "BatchLambdaStack" + "ParameterValue": "digiroad-batch-lambda-stack" }, { "ParameterKey": "Subnet1", diff --git a/aws/cloud-formation/fargateService/prod/README.md b/aws/cloud-formation/fargateService/prod/README.md index dc12cb6907..3cb923f12a 100644 --- a/aws/cloud-formation/fargateService/prod/README.md +++ b/aws/cloud-formation/fargateService/prod/README.md @@ -89,6 +89,32 @@ aws cloudformation create-stack \ --parameters file://aws/cloud-formation/fargateService/prod/PROD-alb-ecs-parameter.json ``` +##Eräajoja varten tuotantotilille luotavat resurssit + +### Luo S3 Bucket lambdan koodia varten +``` +aws cloudformation create-stack \ +--stack-name [esim. digiroad-batch-lambda-bucket] \ +--template-body file://aws/cloud-formation/batchSystem/batchLambda/cicd/prodBatchLambdaDeploymentBucket.yaml +``` +S3-Bucketin luonnin jälkeen pyydä kehitystiimiä toimittamaan lambdan koodi .zip tiedostona sinne + +### Luo Lambda +``` +aws cloudformation create-stack \ +--stack-name digiroad-batch-lambda-stack \ +--template-body file://aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml \ +--parameters file://aws/cloud-formation/batchSystem/batchLambda/prod-batch-lambda-parameter.json +``` + +### Luo eräajoympäristö +``` +aws cloudformation create-stack \ +--stack-name [esim. digiroad-batch-system] \ +--template-body file://aws/cloud-formation/batchSystem/batchSystem.yaml \ +--parameters file://aws/cloud-formation/batchSystem/prod-batch-system-parameter.json +``` + # Ympäristön päivitys **HUOM tarkista ennen jokaista update-stack komentoa parametritiedostojen sisältö** From e733faae4aec76c9bf25a4fc6397cb8aedbf7624 Mon Sep 17 00:00:00 2001 From: "antti.ahopelto" Date: Wed, 11 May 2022 08:27:06 +0300 Subject: [PATCH 04/37] DROTH-3214 Divided pipeline into two. Added tags, added instructions for creating and updating job def --- .../batchSystem/batchLambda/batchLambda.yaml | 12 ++ .../batchLambda/cicd/batchLambdaBuildSpec.yml | 21 ---- ... dev-batch-lambda-pipeline-parameter.json} | 4 + ...line.yaml => dev-batchLambdaPipeline.yaml} | 41 ++++-- .../prod-batch-lambda-pipeline-parameter.json | 14 +++ .../cicd/prod-batchLambdaPipeline.yaml | 119 ++++++++++++++++++ .../dev-batch-lambda-parameter.json | 4 + .../prod-batch-lambda-parameter.json | 4 + .../batchSystem/batchSystem.yaml | 86 ++++++++++++- .../fargateService/prod/README.md | 15 +++ 10 files changed, 287 insertions(+), 33 deletions(-) delete mode 100644 aws/cloud-formation/batchSystem/batchLambda/cicd/batchLambdaBuildSpec.yml rename aws/cloud-formation/batchSystem/batchLambda/cicd/{batch-lambda-pipeline-parameter.json => dev-batch-lambda-pipeline-parameter.json} (81%) rename aws/cloud-formation/batchSystem/batchLambda/cicd/{batchLambdaPipeline.yaml => dev-batchLambdaPipeline.yaml} (81%) create mode 100644 aws/cloud-formation/batchSystem/batchLambda/cicd/prod-batch-lambda-pipeline-parameter.json create mode 100644 aws/cloud-formation/batchSystem/batchLambda/cicd/prod-batchLambdaPipeline.yaml diff --git a/aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml b/aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml index 7e1331abed..a7c7e6bb9a 100644 --- a/aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml +++ b/aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml @@ -6,6 +6,8 @@ Parameters: S3ObjectKey: Description: Key of lambda deployment object Type: String + AccountName: + Description: Name of AWS account dev or prod Resources: BatchLambda: @@ -14,6 +16,11 @@ Resources: Code: S3Bucket: !ImportValue 'BatchLambda-deployment-bucket' S3Key: batch-lambda-deployment + Tags: + - Key: Name + Value: !Join [ '-', [ !Ref AccountName, 'batchLambda' ] ] + - Key: Environment + Value: !Ref AccountName FunctionName: "Batch-Add-Jobs-To-Queue" Handler: AddJobToQueue.handler Role: !GetAtt BatchLambdaRole.Arn @@ -36,6 +43,11 @@ Resources: - 'batch:DescribeJobs' - 'batch:SubmitJob' - 'batch:ListJobs' + Tags: + - Key: Name + Value: !Join [ '-', [ !Ref AccountName, 'batchLambdaRole' ] ] + - Key: Environment + Value: !Ref AccountName Outputs: BatchLambdaOutput: diff --git a/aws/cloud-formation/batchSystem/batchLambda/cicd/batchLambdaBuildSpec.yml b/aws/cloud-formation/batchSystem/batchLambda/cicd/batchLambdaBuildSpec.yml deleted file mode 100644 index 6d110385c2..0000000000 --- a/aws/cloud-formation/batchSystem/batchLambda/cicd/batchLambdaBuildSpec.yml +++ /dev/null @@ -1,21 +0,0 @@ -version: 0.2 -phases: - install: - runtime-versions: - nodejs: 14.x - commands: - - mkdir -p ./lib - - npm install --prefix ./lib aws-sdk - build: - commands: - - cd lib - - zip -r9 ../deployment_package.zip . - - cd .. - - zip -g deployment_package.zip AddJobToQueue.js - - echo Pushing deployment_package.zip to Prod and Dev S3 buckets - - aws s3api put-object --bucket $DEV_BUCKET --key batch-lambda-deployment --body deployment_package.zip - - aws s3api put-object --bucket $PROD_BUCKET --key batch-lambda-deployment --body deployment_package.zip --acl bucket-owner-full-control - post_build: - commands: - - echo Updating Dev account lambda code - - aws lambda update-function-code --function-name Batch-Add-Jobs-To-Queue --zip-file fileb://deployment_package.zip \ No newline at end of file diff --git a/aws/cloud-formation/batchSystem/batchLambda/cicd/batch-lambda-pipeline-parameter.json b/aws/cloud-formation/batchSystem/batchLambda/cicd/dev-batch-lambda-pipeline-parameter.json similarity index 81% rename from aws/cloud-formation/batchSystem/batchLambda/cicd/batch-lambda-pipeline-parameter.json rename to aws/cloud-formation/batchSystem/batchLambda/cicd/dev-batch-lambda-pipeline-parameter.json index 0db6bd197f..f3b1c62163 100644 --- a/aws/cloud-formation/batchSystem/batchLambda/cicd/batch-lambda-pipeline-parameter.json +++ b/aws/cloud-formation/batchSystem/batchLambda/cicd/dev-batch-lambda-pipeline-parameter.json @@ -7,6 +7,10 @@ "ParameterKey": "ProdBucketName", "ParameterValue": "prod-batch-lambda-deployment-bucket" }, + { + "ParameterKey": "ObjectKey", + "ParameterValue": "deployment_package.zip" + }, { "ParameterKey": "CodeCommitRepositoryName", "ParameterValue": "Batch-Lambda" diff --git a/aws/cloud-formation/batchSystem/batchLambda/cicd/batchLambdaPipeline.yaml b/aws/cloud-formation/batchSystem/batchLambda/cicd/dev-batchLambdaPipeline.yaml similarity index 81% rename from aws/cloud-formation/batchSystem/batchLambda/cicd/batchLambdaPipeline.yaml rename to aws/cloud-formation/batchSystem/batchLambda/cicd/dev-batchLambdaPipeline.yaml index 2ed8fee710..7661189369 100644 --- a/aws/cloud-formation/batchSystem/batchLambda/cicd/batchLambdaPipeline.yaml +++ b/aws/cloud-formation/batchSystem/batchLambda/cicd/dev-batchLambdaPipeline.yaml @@ -6,6 +6,9 @@ Parameters: ProdBucketName: Description: Name of prod account S3 bucket for lambda deployment Type: String + ObjectKey: + Description: Key of S3 object for lambda code + Type: String CodeCommitRepositoryName: Description: Name of CodeCommit repo for lambda source code Type: String @@ -22,7 +25,7 @@ Resources: BatchLambdaPipeline: Type: AWS::CodePipeline::Pipeline Properties: - Name: 'batch-lambda-pipeline' + Name: 'dev-batch-lambda-pipeline' RoleArn: !GetAtt PipelineRole.Arn RestartExecutionOnUpdate: true Stages: @@ -60,7 +63,7 @@ Resources: PipelineRole: Type: 'AWS::IAM::Role' Properties: - RoleName: 'batch-lambda-pipeline-role' + RoleName: 'dev-batch-lambda-pipeline-role' AssumeRolePolicyDocument: Version: 2012-10-17 Statement: @@ -71,7 +74,7 @@ Resources: Action: 'sts:AssumeRole' Path: / Policies: - - PolicyName: 'batch-lambda-pipeline-policy' + - PolicyName: 'dev-batch-lambda-pipeline-policy' PolicyDocument: Version: 2012-10-17 Statement: @@ -134,7 +137,7 @@ Resources: CodeBuild: Type: AWS::CodeBuild::Project Properties: - Name: 'batch-lambda-codebuild' + Name: 'dev-batch-lambda-codebuild' Description: 'CodeBuild for batch-lambda' ServiceRole: !Ref CodeBuildRole Environment: @@ -145,20 +148,40 @@ Resources: EnvironmentVariables: - Name: DEV_BUCKET Value: !Ref DevBucketName - - Name: PROD_BUCKET - Value: !Ref ProdBucketName + - Name: OBJECT_KEY + Value: !Ref ObjectKey Artifacts: Name: 'batch-lambda-build-artifact' Type: CODEPIPELINE Source: Type: CODEPIPELINE - BuildSpec: 'batchLambdaBuildSpec.yml' + BuildSpec: | + version: 0.2 + phases: + install: + runtime-versions: + nodejs: 14.x + commands: + - mkdir -p ./lib + - npm install --prefix ./lib aws-sdk + build: + commands: + - cd lib + - zip -r9 ../deployment_package.zip . + - cd .. + - zip -g deployment_package.zip AddJobToQueue.js + - echo Pushing deployment_package.zip to Dev S3 bucket + - aws s3api put-object --bucket $DEV_BUCKET --key $OBJECT_KEY --body deployment_package.zip + post_build: + commands: + - echo Updating Dev account lambda code + - aws lambda update-function-code --function-name Batch-Add-Jobs-To-Queue --zip-file fileb://deployment_package.zip CodeBuildRole: Type: AWS::IAM::Role Properties: Path: / - RoleName: "batch-lambda-codebuild-role" + RoleName: "dev-batch-lambda-codebuild-role" AssumeRolePolicyDocument: Statement: - Effect: "Allow" @@ -167,7 +190,7 @@ Resources: Action: - "sts:AssumeRole" Policies: - - PolicyName: "batch-lambda-codebuild-policy" + - PolicyName: "dev-batch-lambda-codebuild-policy" PolicyDocument: Statement: - Effect: "Allow" diff --git a/aws/cloud-formation/batchSystem/batchLambda/cicd/prod-batch-lambda-pipeline-parameter.json b/aws/cloud-formation/batchSystem/batchLambda/cicd/prod-batch-lambda-pipeline-parameter.json new file mode 100644 index 0000000000..120af2f539 --- /dev/null +++ b/aws/cloud-formation/batchSystem/batchLambda/cicd/prod-batch-lambda-pipeline-parameter.json @@ -0,0 +1,14 @@ +[ + { + "ParameterKey": "DevBucketName", + "ParameterValue": "dev-batch-lambda-deployment-bucket" + }, + { + "ParameterKey": "ProdBucketName", + "ParameterValue": "prod-batch-lambda-deployment-bucket" + }, + { + "ParameterKey": "ObjectKey", + "ParameterValue": "deployment_package.zip" + } +] \ No newline at end of file diff --git a/aws/cloud-formation/batchSystem/batchLambda/cicd/prod-batchLambdaPipeline.yaml b/aws/cloud-formation/batchSystem/batchLambda/cicd/prod-batchLambdaPipeline.yaml new file mode 100644 index 0000000000..abacf89a07 --- /dev/null +++ b/aws/cloud-formation/batchSystem/batchLambda/cicd/prod-batchLambdaPipeline.yaml @@ -0,0 +1,119 @@ +AWSTemplateFormatVersion: "2010-09-09" +Parameters: + DevBucketName: + Decription: Name of dev bucket for lambda deployment + Type: String + + ObjectKey: + Description: Key of S3 object for lambda code + Type: String + + ProdBucketName: + Description: Name of prod account S3 bucket for lambda deployment + Type: String + +Resources: + BatchLambdaPipeline: + Type: AWS::CodePipeline::Pipeline + Properties: + Name: 'prod-batch-lambda-pipeline' + RoleArn: !GetAtt PipelineRole.Arn + RestartExecutionOnUpdate: true + Stages: + - Name: Source + Actions: + Name: SourceAction + ActionTypeID: + Category: Source + Owner: AWS + Provider: S3 + Version: 1 + OutputArtifacts: + - Name: SourceOutput + Configuration: + S3Bucket: !Ref DevBucket + S3ObjectKey: !Ref BucketKey + PollForSourceChanges: 'true' + RunOrder: 1 + - Name: Accept + Actions: + - Name: Approve_Lambda_zip_For_Production + ActionTypeId: + Category: Approval + Owner: AWS + Version: 1 + Provider: Manual + - Name: Build + Actions: + Name: BuildAction + InputArtifacts: + - Name: SourceOutput + ActionTypeId: + Category: Build + Owner: AWS + Version: 1 + Provider: CodeBuild + OutputArtifacts: + - Name: Built + Configuration: + ProjectName: !Ref CodeBuild + RunOrder: 1 + + CodeBuild: + Type: AWS::CodeBuild::Project + Properties: + Name: 'prod-batch-lambda-codebuild' + Description: 'CodeBuild for batch-lambda' + ServiceRole: !Ref CodeBuildRole + Environment: + ComputeType: BUILD_GENERAL1_SMALL + Image: aws/codebuild/standard:5.0 + Type: LINUX_CONTAINER + PrivilegedMode: true + EnvironmentVariables: + - Name: PROD_BUCKET + Value: !Ref ProdBucketName + - Name: DEV_BUCKET + Value: !Ref DevBucketName + - Name: OBJECT_KEY + Value: !Ref ObjectKey + Artifacts: + Name: 'batch-lambda-build-artifact' + Type: CODEPIPELINE + Source: + Type: CODEPIPELINE + BuildSpec: | + version: 0.2 + phases: + build: + commands: + - echo Get object from dev bucket + - aws s3api get-object --bucket $DEV_BUCKET --key $OBJECT_KEY deployment_package.zip + post_build: + commands: + - echo Push object to prod S3 bucket + - aws s3api put-object --bucket $PROD_BUCKET --key $OBJECT_KEY --body deployment_package.zip --acl bucket-owner-full-control + + CodeBuildRole: + Type: AWS::IAM::Role + Properties: + Path: / + RoleName: "prod-batch-lambda-codebuild-role" + AssumeRolePolicyDocument: + Statement: + - Effect: "Allow" + Principal: + Service: "codebuild.amazonaws.com" + Action: + - "sts:AssumeRole" + Policies: + - PolicyName: "prod-batch-lambda-codebuild-policy" + PolicyDocument: + Statement: + - Effect: "Allow" + Resource: + - "*" + Action: + - "s3:GetObject" + - "s3:GetObjectVersion" + - "s3:PutObject" diff --git a/aws/cloud-formation/batchSystem/batchLambda/dev-batch-lambda-parameter.json b/aws/cloud-formation/batchSystem/batchLambda/dev-batch-lambda-parameter.json index bb826bd725..dc9f79a8b3 100644 --- a/aws/cloud-formation/batchSystem/batchLambda/dev-batch-lambda-parameter.json +++ b/aws/cloud-formation/batchSystem/batchLambda/dev-batch-lambda-parameter.json @@ -6,5 +6,9 @@ { "ParameterKey": "S3ObjectKey", "ParameterValue": "batch-lambda-deployment" + }, + { + "ParameterKey": "AccountName", + "ParameterValue": "DEV" } ] \ No newline at end of file diff --git a/aws/cloud-formation/batchSystem/batchLambda/prod-batch-lambda-parameter.json b/aws/cloud-formation/batchSystem/batchLambda/prod-batch-lambda-parameter.json index bb826bd725..9f6b282e9f 100644 --- a/aws/cloud-formation/batchSystem/batchLambda/prod-batch-lambda-parameter.json +++ b/aws/cloud-formation/batchSystem/batchLambda/prod-batch-lambda-parameter.json @@ -6,5 +6,9 @@ { "ParameterKey": "S3ObjectKey", "ParameterValue": "batch-lambda-deployment" + }, + { + "ParameterKey": "AccountName", + "ParameterValue": "Prod" } ] \ No newline at end of file diff --git a/aws/cloud-formation/batchSystem/batchSystem.yaml b/aws/cloud-formation/batchSystem/batchSystem.yaml index a7ec2067fc..932e6d2348 100644 --- a/aws/cloud-formation/batchSystem/batchSystem.yaml +++ b/aws/cloud-formation/batchSystem/batchSystem.yaml @@ -36,6 +36,11 @@ Resources: - Order: 1 ComputeEnvironment: Ref: ComputeEnvironment + Tags: + - Key: Name + Value: !Join [ '-', [ !Ref EnvironmentName, 'JobQueueAdhoc' ] ] + - Key: Environment + Value: !Ref EnvironmentName JobQueue: Type: AWS::Batch::JobQueue @@ -46,6 +51,11 @@ Resources: - Order: 1 ComputeEnvironment: Ref: ComputeEnvironment + Tags: + - Key: Name + Value: !Join [ '-', [ !Ref EnvironmentName, 'JobQueue' ] ] + - Key: Environment + Value: !Ref EnvironmentName ComputeEnvironment: Type: AWS::Batch::ComputeEnvironment @@ -64,6 +74,11 @@ Resources: - !Ref BatchSecurityGroup ServiceRole: Ref: BatchServiceRole + Tags: + - Key: Name + Value: !Join [ '-', [ !Ref EnvironmentName, 'BatchComputeEnvironment' ] ] + - Key: Environment + Value: !Ref EnvironmentName BatchSecurityGroup: Type: AWS::EC2::SecurityGroup @@ -71,6 +86,11 @@ Resources: GroupDescription: Allow inbound to port 80 and 5432, Outbound to everything GroupName: !Join [ '-', [ !Ref EnvironmentName, 'batchSecurityGroup']] VpcId: !Ref VpcIDOfSystem + Tags: + - Key: Name + Value: !Join [ '-', [ !Ref EnvironmentName, 'batchSecurityGroup' ] ] + - Key: Environment + Value: !Ref EnvironmentName BatchTaskRole: Type: AWS::IAM::Role @@ -103,6 +123,11 @@ Resources: - 'ssm:GetParameters' - 'ssm:GetParametersByPath' Resource: '*' + Tags: + - Key: Name + Value: !Join [ '-', [ !Ref EnvironmentName, 'batchTaskRole' ] ] + - Key: Environment + Value: !Ref EnvironmentName BatchServiceRole: Type: AWS::IAM::Role @@ -116,6 +141,11 @@ Resources: Action: sts:AssumeRole ManagedPolicyArns: - arn:aws:iam::aws:policy/service-role/AWSBatchServiceRole + Tags: + - Key: Name + Value: !Join [ '-', [ !Ref EnvironmentName, 'batchServiceRole' ] ] + - Key: Environment + Value: !Ref EnvironmentName BatchSNSTopic: Type: AWS::SNS::Topic @@ -125,7 +155,12 @@ Resources: Subscription: - Endpoint: "kehitys@digiroad.fi" Protocol: "email" - TopicName: !Ref SNSTopicName + TopicName: !Join [ '-', [ !Ref EnvironmentName, !Ref SNSTopicName ] ] + Tags: + - Key: Name + Value: !Join [ '-', [ !Ref EnvironmentName, 'batchFailTopic' ] ] + - Key: Environment + Value: !Ref EnvironmentName BatchEventRule: Type: AWS::Events::Rule @@ -135,7 +170,7 @@ Resources: Name: "BatchEventNotify" State: "ENABLED" Targets: - - Arn: !Join ['', ["arn:aws:sns:eu-west-1:475079312496:", !Ref SNSTopicName]] + - Arn: !Join ['', ["arn:aws:sns:eu-west-1:475079312496:", !Ref EnvironmentName, '-', !Ref SNSTopicName]] Id: "123321" InputTransformer: InputTemplate: | @@ -146,6 +181,11 @@ Resources: name: "$.detail.jobName" queue: "$.detail.jobQueue" reason: "$.detail.statusReason" + Tags: + - Key: Name + Value: !Join [ '-', [ !Ref EnvironmentName, 'batchFailEventRule' ] ] + - Key: Environment + Value: !Ref EnvironmentName #Batch Events RunAnnualBatch: @@ -162,6 +202,11 @@ Resources: !Sub: '${BatchLambdaStackName}-BatchLambdaID' Input: !Join [ '', [ '{ \"jobName\": ', '\"', !Ref EnvironmentName, '\", ', '\"jobDefinition\": ', '\"', !Ref JobDefinitionName, '\", ', '\"type\": \"annual\" }' ]] Id: !Join [ '-' [!Ref EnvironmentName, 'AnnualLambda']] + Tags: + - Key: Name + Value: !Join [ '-', [ !Ref EnvironmentName, 'batchAnnualEventRule' ] ] + - Key: Environment + Value: !Ref EnvironmentName RunMonthlyBatch: Type: AWS::Events::Rule @@ -177,6 +222,11 @@ Resources: !Sub: '${BatchLambdaStackName}-BatchLambdaID' Input: !Join [ '', [ '{ \"jobName\": ', '\"', !Ref EnvironmentName, '\", ', '\"jobDefinition\": ', '\"', !Ref JobDefinitionName, '\", ', '\"type\": \"monthly\" }' ]] Id: !Join [ '-' [!Ref EnvironmentName, 'MonthlyLambda']] + Tags: + - Key: Name + Value: !Join [ '-', [ !Ref EnvironmentName, 'batchMonthlyEventRule' ] ] + - Key: Environment + Value: !Ref EnvironmentName RunWeeklyBatch: Type: AWS::Events::Rule @@ -192,11 +242,16 @@ Resources: !Sub: '${BatchLambdaStackName}-BatchLambdaID' Input: !Join [ '', [ '{ \"jobName\": ', '\"', !Ref EnvironmentName, '\", ', '\"jobDefinition\": ', '\"', !Ref JobDefinitionName, '\", ', '\"type\": \"weekly\" }' ]] Id: !Join [ '-' [!Ref EnvironmentName, 'WeeklyLambda']] + Tags: + - Key: Name + Value: !Join [ '-', [ !Ref EnvironmentName, 'batchWeeklyEventRule' ] ] + - Key: Environment + Value: !Ref EnvironmentName RunDailyBatch: Type: AWS::Events::Rule Properties: - Name: !Join [ '-', [ !Ref EnvironmentName, 'BatchDailyEventRule' ]] + Name: !Join [ '-', [ !Ref EnvironmentName, 'batchDailyEventRule' ]] Description: "Run daily batches" ScheduleExpression: "cron(30 22 ? * 1-5 *)" State: "ENABLED" @@ -207,6 +262,11 @@ Resources: !Sub: '${BatchLambdaStackName}-BatchLambdaID' Input: !Join [ '', [ '{ \"jobName\": ', '\"', !Ref EnvironmentName, '\", ', '\"jobDefinition\": ', '\"', !Ref JobDefinitionName, '\", ', '\"type\": \"daily\" }' ]] Id: !Join [ '-' [!Ref EnvironmentName, 'DailyLambda']] + Tags: + - Key: Name + Value: !Join [ '-', [ !Ref EnvironmentName, 'batchDailyEventRule' ] ] + - Key: Environment + Value: !Ref EnvironmentName #Lambda permissions for events AnnualLambdaPermission: @@ -218,6 +278,11 @@ Resources: !Sub: '${BatchLambdaStackName}-BatchLambdaID' Principal: events.amazonaws.com SourceArn: !GetAtt RunAnnualBatch.Arn + Tags: + - Key: Name + Value: !Join [ '-', [ !Ref EnvironmentName, 'batchAnnualLambdaPermission' ] ] + - Key: Environment + Value: !Ref EnvironmentName MonthlyLambdaPermission: Type: AWS::Lambda::Permission @@ -228,6 +293,11 @@ Resources: !Sub: '${BatchLambdaStackName}-BatchLambdaID' Principal: events.amazonaws.com SourceArn: !GetAtt RunMonthlyBatch.Arn + Tags: + - Key: Name + Value: !Join [ '-', [ !Ref EnvironmentName, 'batchMonthlyLambdaPermission' ] ] + - Key: Environment + Value: !Ref EnvironmentName WeeklyLambdaPermission: Type: AWS::Lambda::Permission @@ -238,6 +308,11 @@ Resources: !Sub: '${BatchLambdaStackName}-BatchLambdaID' Principal: events.amazonaws.com SourceArn: !GetAtt RunWeeklyBatch.Arn + Tags: + - Key: Name + Value: !Join [ '-', [ !Ref EnvironmentName, 'batchWeeklyLambdaPermission' ] ] + - Key: Environment + Value: !Ref EnvironmentName DailyLambdaPermission: Type: AWS::Lambda::Permission @@ -248,6 +323,11 @@ Resources: !Sub: '${BatchLambdaStackName}-BatchLambdaID' Principal: events.amazonaws.com SourceArn: !GetAtt RunDailyBatch.Arn + Tags: + - Key: Name + Value: !Join [ '-', [ !Ref EnvironmentName, 'batchWeeklyLambdaPermission' ] ] + - Key: Environment + Value: !Ref EnvironmentName Outputs: BatchTaskRoleARN: diff --git a/aws/cloud-formation/fargateService/prod/README.md b/aws/cloud-formation/fargateService/prod/README.md index 3cb923f12a..fb0e1613bd 100644 --- a/aws/cloud-formation/fargateService/prod/README.md +++ b/aws/cloud-formation/fargateService/prod/README.md @@ -107,6 +107,13 @@ aws cloudformation create-stack \ --parameters file://aws/cloud-formation/batchSystem/batchLambda/prod-batch-lambda-parameter.json ``` +### Luo JobDefinition tuotantoeräajoja varten +``` +aws batch register-job-definition \ +--profile vaylaapp \ +--region eu-west-1 \ +--cli-input-json file://aws/cloud-formation/batchSystem/ProdBatchJobDefinition.json +``` ### Luo eräajoympäristö ``` aws cloudformation create-stack \ @@ -159,4 +166,12 @@ aws cloudformation update-stack \ --stack-name [esim. digiroad-ALB-ECS] \ --template-body file://aws/cloud-formation/fargateService/alb_ecs.yaml \ --parameters file://aws/cloud-formation/fargateService/prod/PROD-alb-ecs-parameter.json +``` + +### JobDefinition päivitys +``` +aws batch register-job-definition \ +--profile vaylaapp \ +--region eu-west-1 \ +--cli-input-json file://aws/cloud-formation/batchSystem/ProdBatchJobDefinition.json ``` \ No newline at end of file From dfd10488631aa389db3f395f9dad3fafea6815db Mon Sep 17 00:00:00 2001 From: "antti.ahopelto" Date: Wed, 11 May 2022 14:15:19 +0300 Subject: [PATCH 05/37] DROTH-3214 Added owner and project tags, added event rule for RefreshRoadLinkCache batch --- .../batchSystem/batchLambda/batchLambda.yaml | 22 ++- .../dev-batch-lambda-parameter.json | 8 ++ .../prod-batch-lambda-parameter.json | 8 ++ .../batchSystem/batchSystem.yaml | 131 +++++++++++++++--- .../dev-batch-system-parameter.json | 8 ++ .../prod-batch-system-parameter.json | 8 ++ .../qa-batch-system-parameter.json | 8 ++ 7 files changed, 168 insertions(+), 25 deletions(-) diff --git a/aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml b/aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml index a7c7e6bb9a..bfab180ded 100644 --- a/aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml +++ b/aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml @@ -8,6 +8,12 @@ Parameters: Type: String AccountName: Description: Name of AWS account dev or prod + Owner: + Description: "Owner of the of application, that these resources are created for. Used when tagging the resources" + Type: String + Project: + Description: "Project of the of application, that these resources are created for. Used when tagging the resources" + Type: String Resources: BatchLambda: @@ -19,8 +25,12 @@ Resources: Tags: - Key: Name Value: !Join [ '-', [ !Ref AccountName, 'batchLambda' ] ] - - Key: Environment - Value: !Ref AccountName + - Key: Environment + Value: !Ref AccountName + - Key: Owner + Value: !Ref Owner + - Key: Project + Value: !Ref Project FunctionName: "Batch-Add-Jobs-To-Queue" Handler: AddJobToQueue.handler Role: !GetAtt BatchLambdaRole.Arn @@ -46,8 +56,12 @@ Resources: Tags: - Key: Name Value: !Join [ '-', [ !Ref AccountName, 'batchLambdaRole' ] ] - - Key: Environment - Value: !Ref AccountName + - Key: Environment + Value: !Ref AccountName + - Key: Owner + Value: !Ref Owner + - Key: Project + Value: !Ref Project Outputs: BatchLambdaOutput: diff --git a/aws/cloud-formation/batchSystem/batchLambda/dev-batch-lambda-parameter.json b/aws/cloud-formation/batchSystem/batchLambda/dev-batch-lambda-parameter.json index dc9f79a8b3..250f6b991f 100644 --- a/aws/cloud-formation/batchSystem/batchLambda/dev-batch-lambda-parameter.json +++ b/aws/cloud-formation/batchSystem/batchLambda/dev-batch-lambda-parameter.json @@ -10,5 +10,13 @@ { "ParameterKey": "AccountName", "ParameterValue": "DEV" + }, + { + "ParameterKey": "Owner", + "ParameterValue": "vayla" + }, + { + "ParameterKey": "Project", + "ParameterValue": "digiroad2" } ] \ No newline at end of file diff --git a/aws/cloud-formation/batchSystem/batchLambda/prod-batch-lambda-parameter.json b/aws/cloud-formation/batchSystem/batchLambda/prod-batch-lambda-parameter.json index 9f6b282e9f..3f76b1a28e 100644 --- a/aws/cloud-formation/batchSystem/batchLambda/prod-batch-lambda-parameter.json +++ b/aws/cloud-formation/batchSystem/batchLambda/prod-batch-lambda-parameter.json @@ -10,5 +10,13 @@ { "ParameterKey": "AccountName", "ParameterValue": "Prod" + }, + { + "ParameterKey": "Owner", + "ParameterValue": "vayla" + }, + { + "ParameterKey": "Project", + "ParameterValue": "digiroad2" } ] \ No newline at end of file diff --git a/aws/cloud-formation/batchSystem/batchSystem.yaml b/aws/cloud-formation/batchSystem/batchSystem.yaml index 932e6d2348..f3c904c38b 100644 --- a/aws/cloud-formation/batchSystem/batchSystem.yaml +++ b/aws/cloud-formation/batchSystem/batchSystem.yaml @@ -25,6 +25,12 @@ Parameters: Subnet2: Type: String Description: Subnet2 Id for ImportValue + Owner: + Description: "Owner of the of application, that these resources are created for. Used when tagging the resources" + Type: String + Project: + Description: "Project of the of application, that these resources are created for. Used when tagging the resources" + Type: String Resources: JobQueueAdHoc: @@ -41,6 +47,10 @@ Resources: Value: !Join [ '-', [ !Ref EnvironmentName, 'JobQueueAdhoc' ] ] - Key: Environment Value: !Ref EnvironmentName + - Key: Owner + Value: !Ref Owner + - Key: Project + Value: !Ref Project JobQueue: Type: AWS::Batch::JobQueue @@ -56,7 +66,10 @@ Resources: Value: !Join [ '-', [ !Ref EnvironmentName, 'JobQueue' ] ] - Key: Environment Value: !Ref EnvironmentName - + - Key: Owner + Value: !Ref Owner + - Key: Project + Value: !Ref Project ComputeEnvironment: Type: AWS::Batch::ComputeEnvironment Properties: @@ -79,6 +92,10 @@ Resources: Value: !Join [ '-', [ !Ref EnvironmentName, 'BatchComputeEnvironment' ] ] - Key: Environment Value: !Ref EnvironmentName + - Key: Owner + Value: !Ref Owner + - Key: Project + Value: !Ref Project BatchSecurityGroup: Type: AWS::EC2::SecurityGroup @@ -91,6 +108,10 @@ Resources: Value: !Join [ '-', [ !Ref EnvironmentName, 'batchSecurityGroup' ] ] - Key: Environment Value: !Ref EnvironmentName + - Key: Owner + Value: !Ref Owner + - Key: Project + Value: !Ref Project BatchTaskRole: Type: AWS::IAM::Role @@ -128,6 +149,10 @@ Resources: Value: !Join [ '-', [ !Ref EnvironmentName, 'batchTaskRole' ] ] - Key: Environment Value: !Ref EnvironmentName + - Key: Owner + Value: !Ref Owner + - Key: Project + Value: !Ref Project BatchServiceRole: Type: AWS::IAM::Role @@ -146,6 +171,10 @@ Resources: Value: !Join [ '-', [ !Ref EnvironmentName, 'batchServiceRole' ] ] - Key: Environment Value: !Ref EnvironmentName + - Key: Owner + Value: !Ref Owner + - Key: Project + Value: !Ref Project BatchSNSTopic: Type: AWS::SNS::Topic @@ -159,8 +188,12 @@ Resources: Tags: - Key: Name Value: !Join [ '-', [ !Ref EnvironmentName, 'batchFailTopic' ] ] - - Key: Environment - Value: !Ref EnvironmentName + - Key: Environment + Value: !Ref EnvironmentName + - Key: Owner + Value: !Ref Owner + - Key: Project + Value: !Ref Project BatchEventRule: Type: AWS::Events::Rule @@ -184,8 +217,12 @@ Resources: Tags: - Key: Name Value: !Join [ '-', [ !Ref EnvironmentName, 'batchFailEventRule' ] ] - - Key: Environment - Value: !Ref EnvironmentName + - Key: Environment + Value: !Ref EnvironmentName + - Key: Owner + Value: !Ref Owner + - Key: Project + Value: !Ref Project #Batch Events RunAnnualBatch: @@ -205,8 +242,12 @@ Resources: Tags: - Key: Name Value: !Join [ '-', [ !Ref EnvironmentName, 'batchAnnualEventRule' ] ] - - Key: Environment - Value: !Ref EnvironmentName + - Key: Environment + Value: !Ref EnvironmentName + - Key: Owner + Value: !Ref Owner + - Key: Project + Value: !Ref Project RunMonthlyBatch: Type: AWS::Events::Rule @@ -225,8 +266,12 @@ Resources: Tags: - Key: Name Value: !Join [ '-', [ !Ref EnvironmentName, 'batchMonthlyEventRule' ] ] - - Key: Environment - Value: !Ref EnvironmentName + - Key: Environment + Value: !Ref EnvironmentName + - Key: Owner + Value: !Ref Owner + - Key: Project + Value: !Ref Project RunWeeklyBatch: Type: AWS::Events::Rule @@ -245,8 +290,12 @@ Resources: Tags: - Key: Name Value: !Join [ '-', [ !Ref EnvironmentName, 'batchWeeklyEventRule' ] ] - - Key: Environment - Value: !Ref EnvironmentName + - Key: Environment + Value: !Ref EnvironmentName + - Key: Owner + Value: !Ref Owner + - Key: Project + Value: !Ref Project RunDailyBatch: Type: AWS::Events::Rule @@ -265,8 +314,32 @@ Resources: Tags: - Key: Name Value: !Join [ '-', [ !Ref EnvironmentName, 'batchDailyEventRule' ] ] - - Key: Environment - Value: !Ref EnvironmentName + - Key: Environment + Value: !Ref EnvironmentName + +# Need special Event Rule for this batch because batch must be started at specific time + RunRefreshCacheBatch: + Type: AWS::Events::Rule + Properties: + Name: !Join [ '-', [ !Ref EnvironmentName, 'batchRefreshCacheEventRule' ] ] + Description: "Run refresh_road_link_cache batch on Friday 16:00" + ScheduleExpression: "cron(0 0 14 ? * FRI *)" + State: "ENABLED" + Targets: + - Arn: + - !ImportValue + !Sub: '${BatchLambdaStackName}-BatchLambdaID' + Input: !Join [ '', [ '{ \"jobName\": ', '\"', !Ref EnvironmentName, '\", ', '\"jobDefinition\": ', '\"', !Ref JobDefinitionName, '\", ', '\"type\": \"daily\" }' ] ] + Id: !Join [ '-' [ !Ref EnvironmentName, 'DailyLambda' ] ] + Tags: + - Key: Name + Value: !Join [ '-', [ !Ref EnvironmentName, 'batchRefreshCacheEventRule' ] ] + - Key: Environment + Value: !Ref EnvironmentName + - Key: Owner + Value: !Ref Owner + - Key: Project + Value: !Ref Project #Lambda permissions for events AnnualLambdaPermission: @@ -281,8 +354,12 @@ Resources: Tags: - Key: Name Value: !Join [ '-', [ !Ref EnvironmentName, 'batchAnnualLambdaPermission' ] ] - - Key: Environment - Value: !Ref EnvironmentName + - Key: Environment + Value: !Ref EnvironmentName + - Key: Owner + Value: !Ref Owner + - Key: Project + Value: !Ref Project MonthlyLambdaPermission: Type: AWS::Lambda::Permission @@ -296,8 +373,12 @@ Resources: Tags: - Key: Name Value: !Join [ '-', [ !Ref EnvironmentName, 'batchMonthlyLambdaPermission' ] ] - - Key: Environment - Value: !Ref EnvironmentName + - Key: Environment + Value: !Ref EnvironmentName + - Key: Owner + Value: !Ref Owner + - Key: Project + Value: !Ref Project WeeklyLambdaPermission: Type: AWS::Lambda::Permission @@ -311,8 +392,12 @@ Resources: Tags: - Key: Name Value: !Join [ '-', [ !Ref EnvironmentName, 'batchWeeklyLambdaPermission' ] ] - - Key: Environment - Value: !Ref EnvironmentName + - Key: Environment + Value: !Ref EnvironmentName + - Key: Owner + Value: !Ref Owner + - Key: Project + Value: !Ref Project DailyLambdaPermission: Type: AWS::Lambda::Permission @@ -326,8 +411,12 @@ Resources: Tags: - Key: Name Value: !Join [ '-', [ !Ref EnvironmentName, 'batchWeeklyLambdaPermission' ] ] - - Key: Environment - Value: !Ref EnvironmentName + - Key: Environment + Value: !Ref EnvironmentName + - Key: Owner + Value: !Ref Owner + - Key: Project + Value: !Ref Project Outputs: BatchTaskRoleARN: diff --git a/aws/cloud-formation/batchSystem/dev-batch-system-parameter.json b/aws/cloud-formation/batchSystem/dev-batch-system-parameter.json index 87a50225fb..e3de3e9a32 100644 --- a/aws/cloud-formation/batchSystem/dev-batch-system-parameter.json +++ b/aws/cloud-formation/batchSystem/dev-batch-system-parameter.json @@ -30,5 +30,13 @@ { "ParameterKey": "Subnet2", "ParameterValue": "Digiroad-OTH-kehitys-Subnet2-Id" + }, + { + "ParameterKey": "Owner", + "ParameterValue": "vayla" + }, + { + "ParameterKey": "Project", + "ParameterValue": "digiroad2" } ] \ No newline at end of file diff --git a/aws/cloud-formation/batchSystem/prod-batch-system-parameter.json b/aws/cloud-formation/batchSystem/prod-batch-system-parameter.json index df76969da7..5726c65f96 100644 --- a/aws/cloud-formation/batchSystem/prod-batch-system-parameter.json +++ b/aws/cloud-formation/batchSystem/prod-batch-system-parameter.json @@ -30,5 +30,13 @@ { "ParameterKey": "Subnet2", "ParameterValue": "Digiroad-prod-Subnet2-Id" + }, + { + "ParameterKey": "Owner", + "ParameterValue": "vayla" + }, + { + "ParameterKey": "Project", + "ParameterValue": "digiroad2" } ] \ No newline at end of file diff --git a/aws/cloud-formation/batchSystem/qa-batch-system-parameter.json b/aws/cloud-formation/batchSystem/qa-batch-system-parameter.json index 5c8a10ac52..218caeedf1 100644 --- a/aws/cloud-formation/batchSystem/qa-batch-system-parameter.json +++ b/aws/cloud-formation/batchSystem/qa-batch-system-parameter.json @@ -30,5 +30,13 @@ { "ParameterKey": "Subnet2", "ParameterValue": "DigiroadTest-Subnet2-Id" + }, + { + "ParameterKey": "Owner", + "ParameterValue": "vayla" + }, + { + "ParameterKey": "Project", + "ParameterValue": "digiroad2" } ] \ No newline at end of file From 41117470a1215eafd09c43681134aae701334b0d Mon Sep 17 00:00:00 2001 From: "antti.ahopelto" Date: Wed, 11 May 2022 14:27:51 +0300 Subject: [PATCH 06/37] DROTH-3214 Added missing ApplicationName parameter and Tag --- .../batchSystem/batchLambda/batchLambda.yaml | 7 +++- .../dev-batch-lambda-parameter.json | 4 ++ .../prod-batch-lambda-parameter.json | 4 ++ .../batchSystem/batchSystem.yaml | 37 ++++++++++--------- .../dev-batch-system-parameter.json | 4 ++ .../prod-batch-system-parameter.json | 4 ++ .../qa-batch-system-parameter.json | 4 ++ 7 files changed, 45 insertions(+), 19 deletions(-) diff --git a/aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml b/aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml index bfab180ded..422a141dcc 100644 --- a/aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml +++ b/aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml @@ -14,6 +14,9 @@ Parameters: Project: Description: "Project of the of application, that these resources are created for. Used when tagging the resources" Type: String + ApplicationName: + Description: Name of the application (no whitespace or special characters) + Type: String Resources: BatchLambda: @@ -24,7 +27,7 @@ Resources: S3Key: batch-lambda-deployment Tags: - Key: Name - Value: !Join [ '-', [ !Ref AccountName, 'batchLambda' ] ] + Value: !Join [ '-', [ !Ref AccountName, !Ref ApplicationName, 'batchLambda' ] ] - Key: Environment Value: !Ref AccountName - Key: Owner @@ -55,7 +58,7 @@ Resources: - 'batch:ListJobs' Tags: - Key: Name - Value: !Join [ '-', [ !Ref AccountName, 'batchLambdaRole' ] ] + Value: !Join [ '-', [ !Ref AccountName, !Ref ApplicationName, 'batchLambdaRole' ] ] - Key: Environment Value: !Ref AccountName - Key: Owner diff --git a/aws/cloud-formation/batchSystem/batchLambda/dev-batch-lambda-parameter.json b/aws/cloud-formation/batchSystem/batchLambda/dev-batch-lambda-parameter.json index 250f6b991f..ea4dec45be 100644 --- a/aws/cloud-formation/batchSystem/batchLambda/dev-batch-lambda-parameter.json +++ b/aws/cloud-formation/batchSystem/batchLambda/dev-batch-lambda-parameter.json @@ -18,5 +18,9 @@ { "ParameterKey": "Project", "ParameterValue": "digiroad2" + }, + { + "ParameterKey": "ApplicationName", + "ParameterValue": "digiroad2" } ] \ No newline at end of file diff --git a/aws/cloud-formation/batchSystem/batchLambda/prod-batch-lambda-parameter.json b/aws/cloud-formation/batchSystem/batchLambda/prod-batch-lambda-parameter.json index 3f76b1a28e..feb554fee8 100644 --- a/aws/cloud-formation/batchSystem/batchLambda/prod-batch-lambda-parameter.json +++ b/aws/cloud-formation/batchSystem/batchLambda/prod-batch-lambda-parameter.json @@ -18,5 +18,9 @@ { "ParameterKey": "Project", "ParameterValue": "digiroad2" + }, + { + "ParameterKey": "ApplicationName", + "ParameterValue": "digiroad2" } ] \ No newline at end of file diff --git a/aws/cloud-formation/batchSystem/batchSystem.yaml b/aws/cloud-formation/batchSystem/batchSystem.yaml index f3c904c38b..f40772bfeb 100644 --- a/aws/cloud-formation/batchSystem/batchSystem.yaml +++ b/aws/cloud-formation/batchSystem/batchSystem.yaml @@ -31,6 +31,9 @@ Parameters: Project: Description: "Project of the of application, that these resources are created for. Used when tagging the resources" Type: String + ApplicationName: + Description: Name of the application (no whitespace or special characters) + Type: String Resources: JobQueueAdHoc: @@ -44,7 +47,7 @@ Resources: Ref: ComputeEnvironment Tags: - Key: Name - Value: !Join [ '-', [ !Ref EnvironmentName, 'JobQueueAdhoc' ] ] + Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'JobQueueAdhoc' ] ] - Key: Environment Value: !Ref EnvironmentName - Key: Owner @@ -63,7 +66,7 @@ Resources: Ref: ComputeEnvironment Tags: - Key: Name - Value: !Join [ '-', [ !Ref EnvironmentName, 'JobQueue' ] ] + Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'JobQueue' ] ] - Key: Environment Value: !Ref EnvironmentName - Key: Owner @@ -89,7 +92,7 @@ Resources: Ref: BatchServiceRole Tags: - Key: Name - Value: !Join [ '-', [ !Ref EnvironmentName, 'BatchComputeEnvironment' ] ] + Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'BatchComputeEnvironment' ] ] - Key: Environment Value: !Ref EnvironmentName - Key: Owner @@ -105,7 +108,7 @@ Resources: VpcId: !Ref VpcIDOfSystem Tags: - Key: Name - Value: !Join [ '-', [ !Ref EnvironmentName, 'batchSecurityGroup' ] ] + Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'batchSecurityGroup' ] ] - Key: Environment Value: !Ref EnvironmentName - Key: Owner @@ -146,7 +149,7 @@ Resources: Resource: '*' Tags: - Key: Name - Value: !Join [ '-', [ !Ref EnvironmentName, 'batchTaskRole' ] ] + Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'batchTaskRole' ] ] - Key: Environment Value: !Ref EnvironmentName - Key: Owner @@ -168,7 +171,7 @@ Resources: - arn:aws:iam::aws:policy/service-role/AWSBatchServiceRole Tags: - Key: Name - Value: !Join [ '-', [ !Ref EnvironmentName, 'batchServiceRole' ] ] + Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'batchServiceRole' ] ] - Key: Environment Value: !Ref EnvironmentName - Key: Owner @@ -187,7 +190,7 @@ Resources: TopicName: !Join [ '-', [ !Ref EnvironmentName, !Ref SNSTopicName ] ] Tags: - Key: Name - Value: !Join [ '-', [ !Ref EnvironmentName, 'batchFailTopic' ] ] + Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'batchFailTopic' ] ] - Key: Environment Value: !Ref EnvironmentName - Key: Owner @@ -216,7 +219,7 @@ Resources: reason: "$.detail.statusReason" Tags: - Key: Name - Value: !Join [ '-', [ !Ref EnvironmentName, 'batchFailEventRule' ] ] + Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'batchFailEventRule' ] ] - Key: Environment Value: !Ref EnvironmentName - Key: Owner @@ -241,7 +244,7 @@ Resources: Id: !Join [ '-' [!Ref EnvironmentName, 'AnnualLambda']] Tags: - Key: Name - Value: !Join [ '-', [ !Ref EnvironmentName, 'batchAnnualEventRule' ] ] + Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'batchAnnualEventRule' ] ] - Key: Environment Value: !Ref EnvironmentName - Key: Owner @@ -265,7 +268,7 @@ Resources: Id: !Join [ '-' [!Ref EnvironmentName, 'MonthlyLambda']] Tags: - Key: Name - Value: !Join [ '-', [ !Ref EnvironmentName, 'batchMonthlyEventRule' ] ] + Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'batchMonthlyEventRule' ] ] - Key: Environment Value: !Ref EnvironmentName - Key: Owner @@ -289,7 +292,7 @@ Resources: Id: !Join [ '-' [!Ref EnvironmentName, 'WeeklyLambda']] Tags: - Key: Name - Value: !Join [ '-', [ !Ref EnvironmentName, 'batchWeeklyEventRule' ] ] + Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'batchWeeklyEventRule' ] ] - Key: Environment Value: !Ref EnvironmentName - Key: Owner @@ -313,7 +316,7 @@ Resources: Id: !Join [ '-' [!Ref EnvironmentName, 'DailyLambda']] Tags: - Key: Name - Value: !Join [ '-', [ !Ref EnvironmentName, 'batchDailyEventRule' ] ] + Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'batchDailyEventRule' ] ] - Key: Environment Value: !Ref EnvironmentName @@ -333,7 +336,7 @@ Resources: Id: !Join [ '-' [ !Ref EnvironmentName, 'DailyLambda' ] ] Tags: - Key: Name - Value: !Join [ '-', [ !Ref EnvironmentName, 'batchRefreshCacheEventRule' ] ] + Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'batchRefreshCacheEventRule' ] ] - Key: Environment Value: !Ref EnvironmentName - Key: Owner @@ -353,7 +356,7 @@ Resources: SourceArn: !GetAtt RunAnnualBatch.Arn Tags: - Key: Name - Value: !Join [ '-', [ !Ref EnvironmentName, 'batchAnnualLambdaPermission' ] ] + Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'batchAnnualLambdaPermission' ] ] - Key: Environment Value: !Ref EnvironmentName - Key: Owner @@ -372,7 +375,7 @@ Resources: SourceArn: !GetAtt RunMonthlyBatch.Arn Tags: - Key: Name - Value: !Join [ '-', [ !Ref EnvironmentName, 'batchMonthlyLambdaPermission' ] ] + Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'batchMonthlyLambdaPermission' ] ] - Key: Environment Value: !Ref EnvironmentName - Key: Owner @@ -391,7 +394,7 @@ Resources: SourceArn: !GetAtt RunWeeklyBatch.Arn Tags: - Key: Name - Value: !Join [ '-', [ !Ref EnvironmentName, 'batchWeeklyLambdaPermission' ] ] + Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'batchWeeklyLambdaPermission' ] ] - Key: Environment Value: !Ref EnvironmentName - Key: Owner @@ -410,7 +413,7 @@ Resources: SourceArn: !GetAtt RunDailyBatch.Arn Tags: - Key: Name - Value: !Join [ '-', [ !Ref EnvironmentName, 'batchWeeklyLambdaPermission' ] ] + Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'batchWeeklyLambdaPermission' ] ] - Key: Environment Value: !Ref EnvironmentName - Key: Owner diff --git a/aws/cloud-formation/batchSystem/dev-batch-system-parameter.json b/aws/cloud-formation/batchSystem/dev-batch-system-parameter.json index e3de3e9a32..c74f3211e4 100644 --- a/aws/cloud-formation/batchSystem/dev-batch-system-parameter.json +++ b/aws/cloud-formation/batchSystem/dev-batch-system-parameter.json @@ -38,5 +38,9 @@ { "ParameterKey": "Project", "ParameterValue": "digiroad2" + }, + { + "ParameterKey": "ApplicationName", + "ParameterValue": "digiroad2" } ] \ No newline at end of file diff --git a/aws/cloud-formation/batchSystem/prod-batch-system-parameter.json b/aws/cloud-formation/batchSystem/prod-batch-system-parameter.json index 5726c65f96..db9d0c8687 100644 --- a/aws/cloud-formation/batchSystem/prod-batch-system-parameter.json +++ b/aws/cloud-formation/batchSystem/prod-batch-system-parameter.json @@ -38,5 +38,9 @@ { "ParameterKey": "Project", "ParameterValue": "digiroad2" + }, + { + "ParameterKey": "ApplicationName", + "ParameterValue": "digiroad2" } ] \ No newline at end of file diff --git a/aws/cloud-formation/batchSystem/qa-batch-system-parameter.json b/aws/cloud-formation/batchSystem/qa-batch-system-parameter.json index 218caeedf1..984fd43d0c 100644 --- a/aws/cloud-formation/batchSystem/qa-batch-system-parameter.json +++ b/aws/cloud-formation/batchSystem/qa-batch-system-parameter.json @@ -38,5 +38,9 @@ { "ParameterKey": "Project", "ParameterValue": "digiroad2" + }, + { + "ParameterKey": "ApplicationName", + "ParameterValue": "digiroad2" } ] \ No newline at end of file From 0818834f138d89533e18ec9755a5c7c3aff4ed19 Mon Sep 17 00:00:00 2001 From: "antti.ahopelto" Date: Thu, 12 May 2022 08:04:07 +0300 Subject: [PATCH 07/37] DROTH-3214 Changed parameter names --- .../batchSystem/batchLambda/batchLambda.yaml | 10 +++++----- .../cicd/dev-batch-lambda-pipeline-parameter.json | 2 +- .../batchLambda/cicd/dev-batchLambdaPipeline.yaml | 4 ++-- .../cicd/prod-batch-lambda-pipeline-parameter.json | 2 +- .../batchLambda/cicd/prod-batchLambdaPipeline.yaml | 6 +++--- .../batchLambda/dev-batch-lambda-parameter.json | 2 +- .../batchLambda/prod-batch-lambda-parameter.json | 2 +- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml b/aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml index 422a141dcc..1018eceef2 100644 --- a/aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml +++ b/aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml @@ -6,7 +6,7 @@ Parameters: S3ObjectKey: Description: Key of lambda deployment object Type: String - AccountName: + EnvironmentName: Description: Name of AWS account dev or prod Owner: Description: "Owner of the of application, that these resources are created for. Used when tagging the resources" @@ -27,9 +27,9 @@ Resources: S3Key: batch-lambda-deployment Tags: - Key: Name - Value: !Join [ '-', [ !Ref AccountName, !Ref ApplicationName, 'batchLambda' ] ] + Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'batchLambda' ] ] - Key: Environment - Value: !Ref AccountName + Value: !Ref EnvironmentName - Key: Owner Value: !Ref Owner - Key: Project @@ -58,9 +58,9 @@ Resources: - 'batch:ListJobs' Tags: - Key: Name - Value: !Join [ '-', [ !Ref AccountName, !Ref ApplicationName, 'batchLambdaRole' ] ] + Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'batchLambdaRole' ] ] - Key: Environment - Value: !Ref AccountName + Value: !Ref EnvironmentName - Key: Owner Value: !Ref Owner - Key: Project diff --git a/aws/cloud-formation/batchSystem/batchLambda/cicd/dev-batch-lambda-pipeline-parameter.json b/aws/cloud-formation/batchSystem/batchLambda/cicd/dev-batch-lambda-pipeline-parameter.json index f3b1c62163..ce1543b6b0 100644 --- a/aws/cloud-formation/batchSystem/batchLambda/cicd/dev-batch-lambda-pipeline-parameter.json +++ b/aws/cloud-formation/batchSystem/batchLambda/cicd/dev-batch-lambda-pipeline-parameter.json @@ -8,7 +8,7 @@ "ParameterValue": "prod-batch-lambda-deployment-bucket" }, { - "ParameterKey": "ObjectKey", + "ParameterKey": "LambdaCodeS3Key", "ParameterValue": "deployment_package.zip" }, { diff --git a/aws/cloud-formation/batchSystem/batchLambda/cicd/dev-batchLambdaPipeline.yaml b/aws/cloud-formation/batchSystem/batchLambda/cicd/dev-batchLambdaPipeline.yaml index 7661189369..dd25df4b16 100644 --- a/aws/cloud-formation/batchSystem/batchLambda/cicd/dev-batchLambdaPipeline.yaml +++ b/aws/cloud-formation/batchSystem/batchLambda/cicd/dev-batchLambdaPipeline.yaml @@ -6,7 +6,7 @@ Parameters: ProdBucketName: Description: Name of prod account S3 bucket for lambda deployment Type: String - ObjectKey: + LambdaCodeS3Key: Description: Key of S3 object for lambda code Type: String CodeCommitRepositoryName: @@ -149,7 +149,7 @@ Resources: - Name: DEV_BUCKET Value: !Ref DevBucketName - Name: OBJECT_KEY - Value: !Ref ObjectKey + Value: !Ref LambdaCodeS3Key Artifacts: Name: 'batch-lambda-build-artifact' Type: CODEPIPELINE diff --git a/aws/cloud-formation/batchSystem/batchLambda/cicd/prod-batch-lambda-pipeline-parameter.json b/aws/cloud-formation/batchSystem/batchLambda/cicd/prod-batch-lambda-pipeline-parameter.json index 120af2f539..747e043e10 100644 --- a/aws/cloud-formation/batchSystem/batchLambda/cicd/prod-batch-lambda-pipeline-parameter.json +++ b/aws/cloud-formation/batchSystem/batchLambda/cicd/prod-batch-lambda-pipeline-parameter.json @@ -8,7 +8,7 @@ "ParameterValue": "prod-batch-lambda-deployment-bucket" }, { - "ParameterKey": "ObjectKey", + "ParameterKey": "LambdaCodeS3Key", "ParameterValue": "deployment_package.zip" } ] \ No newline at end of file diff --git a/aws/cloud-formation/batchSystem/batchLambda/cicd/prod-batchLambdaPipeline.yaml b/aws/cloud-formation/batchSystem/batchLambda/cicd/prod-batchLambdaPipeline.yaml index abacf89a07..5446c6ea26 100644 --- a/aws/cloud-formation/batchSystem/batchLambda/cicd/prod-batchLambdaPipeline.yaml +++ b/aws/cloud-formation/batchSystem/batchLambda/cicd/prod-batchLambdaPipeline.yaml @@ -4,7 +4,7 @@ Parameters: Decription: Name of dev bucket for lambda deployment Type: String - ObjectKey: + LambdaCodeS3Key: Description: Key of S3 object for lambda code Type: String @@ -32,7 +32,7 @@ Resources: - Name: SourceOutput Configuration: S3Bucket: !Ref DevBucket - S3ObjectKey: !Ref BucketKey + S3ObjectKey: !Ref LambdaCodeS3Key PollForSourceChanges: 'true' RunOrder: 1 - Name: Accept @@ -76,7 +76,7 @@ Resources: - Name: DEV_BUCKET Value: !Ref DevBucketName - Name: OBJECT_KEY - Value: !Ref ObjectKey + Value: !Ref LambdaCodeS3Key Artifacts: Name: 'batch-lambda-build-artifact' Type: CODEPIPELINE diff --git a/aws/cloud-formation/batchSystem/batchLambda/dev-batch-lambda-parameter.json b/aws/cloud-formation/batchSystem/batchLambda/dev-batch-lambda-parameter.json index ea4dec45be..da864a3334 100644 --- a/aws/cloud-formation/batchSystem/batchLambda/dev-batch-lambda-parameter.json +++ b/aws/cloud-formation/batchSystem/batchLambda/dev-batch-lambda-parameter.json @@ -8,7 +8,7 @@ "ParameterValue": "batch-lambda-deployment" }, { - "ParameterKey": "AccountName", + "ParameterKey": "EnvironmentName", "ParameterValue": "DEV" }, { diff --git a/aws/cloud-formation/batchSystem/batchLambda/prod-batch-lambda-parameter.json b/aws/cloud-formation/batchSystem/batchLambda/prod-batch-lambda-parameter.json index feb554fee8..3d4f1f370a 100644 --- a/aws/cloud-formation/batchSystem/batchLambda/prod-batch-lambda-parameter.json +++ b/aws/cloud-formation/batchSystem/batchLambda/prod-batch-lambda-parameter.json @@ -8,7 +8,7 @@ "ParameterValue": "batch-lambda-deployment" }, { - "ParameterKey": "AccountName", + "ParameterKey": "EnvironmentName", "ParameterValue": "Prod" }, { From 1e6b50b918a600b04d91b373f1218302c0a7c632 Mon Sep 17 00:00:00 2001 From: "antti.ahopelto" Date: Thu, 12 May 2022 12:34:48 +0300 Subject: [PATCH 08/37] DROTH-3214 Fixed yaml syntax errors in lambda pipeline --- .../cicd/dev-batchLambdaPipeline.yaml | 130 ++++++++++-------- 1 file changed, 72 insertions(+), 58 deletions(-) diff --git a/aws/cloud-formation/batchSystem/batchLambda/cicd/dev-batchLambdaPipeline.yaml b/aws/cloud-formation/batchSystem/batchLambda/cicd/dev-batchLambdaPipeline.yaml index dd25df4b16..5469bb5808 100644 --- a/aws/cloud-formation/batchSystem/batchLambda/cicd/dev-batchLambdaPipeline.yaml +++ b/aws/cloud-formation/batchSystem/batchLambda/cicd/dev-batchLambdaPipeline.yaml @@ -22,73 +22,87 @@ Resources: Properties: BucketName: !Ref DevBucketName + LambdaArtifactStore: + Type: AWS::S3::Bucket + Properties: + BucketName: 'batch-lambda-build-artifact-store' + BatchLambdaPipeline: Type: AWS::CodePipeline::Pipeline Properties: Name: 'dev-batch-lambda-pipeline' RoleArn: !GetAtt PipelineRole.Arn - RestartExecutionOnUpdate: true Stages: - - Name: Source + - + Name: Source Actions: - Name: SourceAction - ActionTypeID: - Category: Source - Owner: AWS - Provider: CodeCommit - Version: 1 - OutputArtifacts: - - Name: SourceOutput - Configuration: - RepositoryName: !Ref CodeCommitRepositoryName - BranchName: !Ref CodeCommitBranch - PollForSourceChanges: false - RunOrder: 1 - - Name: Build + - + Name: SourceAction + ActionTypeId: + Category: Source + Owner: AWS + Version: 1 + Provider: CodeCommit + OutputArtifacts: + - + Name: SourceOutput + Configuration: + RepositoryName: !Ref CodeCommitRepositoryName + BranchName: !Ref CodeCommitBranch + PollForSourceChanges: false + RunOrder: 1 + - + Name: Build Actions: - Name: BuildAction - InputArtifacts: - - Name: SourceOutput - ActionTypeId: - Category: Build - Owner: AWS - Version: 1 - Provider: CodeBuild - OutputArtifacts: - - Name: Built - Configuration: - ProjectName: !Ref CodeBuild - RunOrder: 1 + - + Name: BuildAction + InputArtifacts: + - + Name: SourceOutput + ActionTypeId: + Category: Build + Owner: AWS + Version: 1 + Provider: CodeBuild + OutputArtifacts: + - + Name: Built + Configuration: + ProjectName: !Ref CodeBuild + RunOrder: 1 + ArtifactStore: + Type: S3 + Location: !Ref LambdaArtifactStore PipelineRole: Type: 'AWS::IAM::Role' - Properties: - RoleName: 'dev-batch-lambda-pipeline-role' - AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Principal: - Service: - - codepipeline.amazonaws.com - Action: 'sts:AssumeRole' - Path: / - Policies: - - PolicyName: 'dev-batch-lambda-pipeline-policy' - PolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Action: - - 'codebuild:BatchGetBuilds' - - 'codebuild:StartBuild' - Resource: '*' - - Effect: Allow - Action: - - 'lambda:*' - - 'codebuild:*' - - 's3:*' - Resource: '*' + Properties: + RoleName: 'dev-batch-lambda-pipeline-role' + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - codepipeline.amazonaws.com + Action: 'sts:AssumeRole' + Path: / + Policies: + - PolicyName: 'dev-batch-lambda-pipeline-policy' + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - 'codebuild:BatchGetBuilds' + - 'codebuild:StartBuild' + Resource: '*' + - Effect: Allow + Action: + - 'lambda:*' + - 'codebuild:*' + - 's3:*' + Resource: '*' StartPipeLineEventRole: Type: AWS::IAM::Role @@ -109,7 +123,7 @@ Resources: Statement: - Effect: Allow Action: codepipeline:StartPipelineExecution - Resource: !Ref BatchLambdaPipeline + Resource: !Sub arn:aws:codepipeline:${AWS::Region}:475079312496:${BatchLambdaPipeline} StartPipelineRule: Type: AWS::Events::Rule @@ -130,7 +144,7 @@ Resources: referenceName: - master Targets: - - Arn: !GetAtt BatchLambdaPipeline.Arn + - Arn: !Sub arn:aws:codepipeline:${AWS::Region}:475079312496:${BatchLambdaPipeline} RoleArn: !GetAtt StartPipeLineEventRole.Arn Id: batch-lambda-pipeline From 8524f8124583803de30371d85056e9c53ed0783e Mon Sep 17 00:00:00 2001 From: "antti.ahopelto" Date: Mon, 16 May 2022 09:35:45 +0300 Subject: [PATCH 09/37] DROTH-3214 Fixed yaml syntax errors in batch system, added SecurityGroup rules, renamed resources --- .../batchSystem/batchLambda/batchLambda.yaml | 58 ++-- .../cicd/dev-batchLambdaPipeline.yaml | 9 +- .../dev-batch-lambda-parameter.json | 2 +- .../batchSystem/batchSystem.yaml | 258 ++++++------------ .../dev-batch-system-parameter.json | 22 +- .../prod-batch-system-parameter.json | 22 +- .../qa-batch-system-parameter.json | 22 +- 7 files changed, 196 insertions(+), 197 deletions(-) diff --git a/aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml b/aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml index 1018eceef2..d79758655e 100644 --- a/aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml +++ b/aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml @@ -8,6 +8,7 @@ Parameters: Type: String EnvironmentName: Description: Name of AWS account dev or prod + Type: String Owner: Description: "Owner of the of application, that these resources are created for. Used when tagging the resources" Type: String @@ -23,21 +24,21 @@ Resources: Type: AWS::Lambda::Function Properties: Code: - S3Bucket: !ImportValue 'BatchLambda-deployment-bucket' - S3Key: batch-lambda-deployment - Tags: - - Key: Name - Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'batchLambda' ] ] - - Key: Environment - Value: !Ref EnvironmentName - - Key: Owner - Value: !Ref Owner - - Key: Project - Value: !Ref Project - FunctionName: "Batch-Add-Jobs-To-Queue" - Handler: AddJobToQueue.handler - Role: !GetAtt BatchLambdaRole.Arn - Runtime: nodejs14.x + S3Bucket: !ImportValue 'batch-lambda-deployment-bucket' + S3Key: !Ref S3ObjectKey + Tags: + - Key: Name + Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'batchLambda' ] ] + - Key: Environment + Value: !Ref EnvironmentName + - Key: Owner + Value: !Ref Owner + - Key: Project + Value: !Ref Project + FunctionName: "Dev-Batch-Add-Jobs-To-Queue" + Handler: AddJobToQueue.handler + Role: !GetAtt BatchLambdaRole.Arn + Runtime: nodejs14.x BatchLambdaRole: Type: AWS::IAM::Role @@ -45,17 +46,24 @@ Resources: AssumeRolePolicyDocument: Version: "2012-10-17" Statement: - - Effect: Allow + - Effect: "Allow" Principal: - Service: - - lambda.amazonaws.com + Service: "lambda.amazonaws.com" Action: - 'sts:AssumeRole' - - 'logs:CreateLogStream' - - 'logs:PutLogEvents' - - 'batch:DescribeJobs' - - 'batch:SubmitJob' - - 'batch:ListJobs' + Policies: + - PolicyName: "dev-batch-lambda-policy" + PolicyDocument: + Statement: + - Effect: "Allow" + Resource: "*" + Action: + - 'logs:CreateLogStream' + - 'logs:PutLogEvents' + - 'batch:DescribeJobs' + - 'batch:SubmitJob' + - 'batch:ListJobs' + Tags: - Key: Name Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'batchLambdaRole' ] ] @@ -69,6 +77,6 @@ Resources: Outputs: BatchLambdaOutput: Description: Arn of lambda for adding jobs to queue - Value: !Ref BatchLambda + Value: !GetAtt BatchLambda.Arn Export: - Name: !Sub "${AWS::StackName}-BatchLambdaID" \ No newline at end of file + Name: !Sub "BatchLambdaID" \ No newline at end of file diff --git a/aws/cloud-formation/batchSystem/batchLambda/cicd/dev-batchLambdaPipeline.yaml b/aws/cloud-formation/batchSystem/batchLambda/cicd/dev-batchLambdaPipeline.yaml index 5469bb5808..368e25f5fe 100644 --- a/aws/cloud-formation/batchSystem/batchLambda/cicd/dev-batchLambdaPipeline.yaml +++ b/aws/cloud-formation/batchSystem/batchLambda/cicd/dev-batchLambdaPipeline.yaml @@ -239,4 +239,11 @@ Resources: Action: - "s3:GetObject" - "s3:GetObjectVersion" - - "s3:PutObject" \ No newline at end of file + - "s3:PutObject" + +Outputs: + LambdaDeploymentBucketOutput: + Description: Deployment bucket for batch lambda dev account + Value: !Ref LambdaDeploymentBucket + Export: + Name: 'batch-lambda-deployment-bucket' \ No newline at end of file diff --git a/aws/cloud-formation/batchSystem/batchLambda/dev-batch-lambda-parameter.json b/aws/cloud-formation/batchSystem/batchLambda/dev-batch-lambda-parameter.json index da864a3334..7da15ced7c 100644 --- a/aws/cloud-formation/batchSystem/batchLambda/dev-batch-lambda-parameter.json +++ b/aws/cloud-formation/batchSystem/batchLambda/dev-batch-lambda-parameter.json @@ -5,7 +5,7 @@ }, { "ParameterKey": "S3ObjectKey", - "ParameterValue": "batch-lambda-deployment" + "ParameterValue": "deployment_package.zip" }, { "ParameterKey": "EnvironmentName", diff --git a/aws/cloud-formation/batchSystem/batchSystem.yaml b/aws/cloud-formation/batchSystem/batchSystem.yaml index f40772bfeb..b3458834be 100644 --- a/aws/cloud-formation/batchSystem/batchSystem.yaml +++ b/aws/cloud-formation/batchSystem/batchSystem.yaml @@ -34,50 +34,49 @@ Parameters: ApplicationName: Description: Name of the application (no whitespace or special characters) Type: String + LambdaInputAnnual: + Description: Input for adding annual jobs to queue + Type: String + LambdaInputMonthly: + Description: Input for adding monthly jobs to queue + Type: String + LambdaInputWeekly: + Description: Input for adding annual jobs to queue + Type: String + LambdaInputDaily: + Description: Input for adding annual jobs to queue + Type: String + LambdaInputRefreshCache: + Description: Input for adding refresh_roadlinks job to queue + Type: String Resources: JobQueueAdHoc: Type: AWS::Batch::JobQueue Properties: - JobQueueName: !Join [ '-', [ !Ref EnvironmentName, 'adhoc']] + JobQueueName: !Join [ '-', [ !Ref EnvironmentName, 'batch-adhoc']] Priority: 1 ComputeEnvironmentOrder: - Order: 1 ComputeEnvironment: Ref: ComputeEnvironment - Tags: - - Key: Name - Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'JobQueueAdhoc' ] ] - - Key: Environment - Value: !Ref EnvironmentName - - Key: Owner - Value: !Ref Owner - - Key: Project - Value: !Ref Project + Tags: {"Name" : "Digiroad2-JobQueue-Adhoc", "Environment" : !Ref EnvironmentName, "Owner": !Ref Owner, "Project": !Ref Project} JobQueue: Type: AWS::Batch::JobQueue Properties: - JobQueueName: !Join [ '-', [ !Ref EnvironmentName, 'jobQueue']] + JobQueueName: !Join [ '-', [ !Ref EnvironmentName, 'batch-jobQueue']] Priority: 1 ComputeEnvironmentOrder: - Order: 1 ComputeEnvironment: Ref: ComputeEnvironment - Tags: - - Key: Name - Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'JobQueue' ] ] - - Key: Environment - Value: !Ref EnvironmentName - - Key: Owner - Value: !Ref Owner - - Key: Project - Value: !Ref Project + Tags: {"Name" : "Digiroad2-JobQueue", "Environment" : !Ref EnvironmentName, "Owner": !Ref Owner, "Project": !Ref Project} ComputeEnvironment: Type: AWS::Batch::ComputeEnvironment Properties: Type: MANAGED - ComputeEnvironmentName: !Join [ '-', [ !Ref EnvironmentName, 'BatchCompute']] + ComputeEnvironmentName: !Join [ '-', [ !Ref EnvironmentName, 'BatchComputeEnvironment']] ComputeResources: Type: FARGATE MaxvCpus: 32 @@ -87,24 +86,44 @@ Resources: - Fn::ImportValue: !Ref Subnet2 SecurityGroupIds: - - !Ref BatchSecurityGroup + - 'sg-011055fc8eac78b1d' ServiceRole: Ref: BatchServiceRole - Tags: - - Key: Name - Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'BatchComputeEnvironment' ] ] - - Key: Environment - Value: !Ref EnvironmentName - - Key: Owner - Value: !Ref Owner - - Key: Project - Value: !Ref Project + Tags: {"Name" : "Digiroad2-BatchComputeEnvironment", "Environment" : !Ref EnvironmentName, "Owner": !Ref Owner, "Project": !Ref Project} BatchSecurityGroup: Type: AWS::EC2::SecurityGroup Properties: - GroupDescription: Allow inbound to port 80 and 5432, Outbound to everything - GroupName: !Join [ '-', [ !Ref EnvironmentName, 'batchSecurityGroup']] + GroupDescription: "Allow inbound to port 80, 443 and 5432, Outbound to everything" + GroupName: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName,'batchSecurityGroup']] + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 80 + ToPort: 80 + CidrIp: 0.0.0.0/0 + - IpProtocol: tcp + FromPort: 80 + ToPort: 80 + CidrIpv6: ::/0 + - IpProtocol: tcp + FromPort: 5432 + ToPort: 5432 + CidrIp: 0.0.0.0/0 + - IpProtocol: tcp + FromPort: 5432 + ToPort: 5432 + CidrIpv6: ::/0 + - IpProtocol: tcp + FromPort: 443 + ToPort: 443 + CidrIp: 0.0.0.0/0 + - IpProtocol: tcp + FromPort: 443 + ToPort: 443 + CidrIpv6: ::/0 + SecurityGroupEgress: + - IpProtocol: "-1" + CidrIp: 0.0.0.0/0 VpcId: !Ref VpcIDOfSystem Tags: - Key: Name @@ -127,7 +146,7 @@ Resources: Action: ['sts:AssumeRole'] Path: / Policies: - - PolicyName: !Join [ '-', [ !Ref EnvironmentName, 'AmazonBatchECSTaskExecutionRolePolicy']] + - PolicyName: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName,'AmazonBatchECSTaskExecutionRolePolicy']] PolicyDocument: Statement: - Effect: Allow @@ -189,21 +208,21 @@ Resources: Protocol: "email" TopicName: !Join [ '-', [ !Ref EnvironmentName, !Ref SNSTopicName ] ] Tags: - - Key: Name - Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'batchFailTopic' ] ] - - Key: Environment - Value: !Ref EnvironmentName - - Key: Owner - Value: !Ref Owner - - Key: Project - Value: !Ref Project + - Key: Name + Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'batchFailTopic' ] ] + - Key: Environment + Value: !Ref EnvironmentName + - Key: Owner + Value: !Ref Owner + - Key: Project + Value: !Ref Project BatchEventRule: Type: AWS::Events::Rule Properties: Description: "Send SNS email for failed batch jobs" EventPattern: {"detail-type": ["Batch Job State Change"], "source": ["aws.batch"], "detail": {"status": ["FAILED"]}} - Name: "BatchEventNotify" + Name: !Join [ '-', [ !Ref EnvironmentName, 'batch-event-notify' ]] State: "ENABLED" Targets: - Arn: !Join ['', ["arn:aws:sns:eu-west-1:475079312496:", !Ref EnvironmentName, '-', !Ref SNSTopicName]] @@ -217,15 +236,6 @@ Resources: name: "$.detail.jobName" queue: "$.detail.jobQueue" reason: "$.detail.statusReason" - Tags: - - Key: Name - Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'batchFailEventRule' ] ] - - Key: Environment - Value: !Ref EnvironmentName - - Key: Owner - Value: !Ref Owner - - Key: Project - Value: !Ref Project #Batch Events RunAnnualBatch: @@ -237,20 +247,9 @@ Resources: State: "ENABLED" Targets: - - Arn: - - !ImportValue - !Sub: '${BatchLambdaStackName}-BatchLambdaID' - Input: !Join [ '', [ '{ \"jobName\": ', '\"', !Ref EnvironmentName, '\", ', '\"jobDefinition\": ', '\"', !Ref JobDefinitionName, '\", ', '\"type\": \"annual\" }' ]] - Id: !Join [ '-' [!Ref EnvironmentName, 'AnnualLambda']] - Tags: - - Key: Name - Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'batchAnnualEventRule' ] ] - - Key: Environment - Value: !Ref EnvironmentName - - Key: Owner - Value: !Ref Owner - - Key: Project - Value: !Ref Project + Arn: !ImportValue 'BatchLambdaID' + Input: !Ref LambdaInputAnnual + Id: !Join [ '-' , [!Ref EnvironmentName, 'AnnualLambda']] RunMonthlyBatch: Type: AWS::Events::Rule @@ -261,20 +260,9 @@ Resources: State: "ENABLED" Targets: - - Arn: - - !ImportValue - !Sub: '${BatchLambdaStackName}-BatchLambdaID' - Input: !Join [ '', [ '{ \"jobName\": ', '\"', !Ref EnvironmentName, '\", ', '\"jobDefinition\": ', '\"', !Ref JobDefinitionName, '\", ', '\"type\": \"monthly\" }' ]] - Id: !Join [ '-' [!Ref EnvironmentName, 'MonthlyLambda']] - Tags: - - Key: Name - Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'batchMonthlyEventRule' ] ] - - Key: Environment - Value: !Ref EnvironmentName - - Key: Owner - Value: !Ref Owner - - Key: Project - Value: !Ref Project + Arn: !ImportValue 'BatchLambdaID' + Input: !Ref LambdaInputMonthly + Id: !Join [ '-' , [!Ref EnvironmentName, 'MonthlyLambda']] RunWeeklyBatch: Type: AWS::Events::Rule @@ -285,20 +273,9 @@ Resources: State: "ENABLED" Targets: - - Arn: - - !ImportValue - !Sub: '${BatchLambdaStackName}-BatchLambdaID' - Input: !Join [ '', [ '{ \"jobName\": ', '\"', !Ref EnvironmentName, '\", ', '\"jobDefinition\": ', '\"', !Ref JobDefinitionName, '\", ', '\"type\": \"weekly\" }' ]] - Id: !Join [ '-' [!Ref EnvironmentName, 'WeeklyLambda']] - Tags: - - Key: Name - Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'batchWeeklyEventRule' ] ] - - Key: Environment - Value: !Ref EnvironmentName - - Key: Owner - Value: !Ref Owner - - Key: Project - Value: !Ref Project + Arn: !ImportValue 'BatchLambdaID' + Input: !Ref LambdaInputWeekly + Id: !Join [ '-' , [!Ref EnvironmentName, 'WeeklyLambda']] RunDailyBatch: Type: AWS::Events::Rule @@ -309,16 +286,9 @@ Resources: State: "ENABLED" Targets: - - Arn: - - !ImportValue - !Sub: '${BatchLambdaStackName}-BatchLambdaID' - Input: !Join [ '', [ '{ \"jobName\": ', '\"', !Ref EnvironmentName, '\", ', '\"jobDefinition\": ', '\"', !Ref JobDefinitionName, '\", ', '\"type\": \"daily\" }' ]] - Id: !Join [ '-' [!Ref EnvironmentName, 'DailyLambda']] - Tags: - - Key: Name - Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'batchDailyEventRule' ] ] - - Key: Environment - Value: !Ref EnvironmentName + Arn: !ImportValue 'BatchLambdaID' + Input: !Ref LambdaInputDaily + Id: !Join [ '-' , [!Ref EnvironmentName, 'DailyLambda']] # Need special Event Rule for this batch because batch must be started at specific time RunRefreshCacheBatch: @@ -326,100 +296,54 @@ Resources: Properties: Name: !Join [ '-', [ !Ref EnvironmentName, 'batchRefreshCacheEventRule' ] ] Description: "Run refresh_road_link_cache batch on Friday 16:00" - ScheduleExpression: "cron(0 0 14 ? * FRI *)" + ScheduleExpression: "cron(0 14 ? * FRI *)" State: "ENABLED" Targets: - - Arn: - - !ImportValue - !Sub: '${BatchLambdaStackName}-BatchLambdaID' - Input: !Join [ '', [ '{ \"jobName\": ', '\"', !Ref EnvironmentName, '\", ', '\"jobDefinition\": ', '\"', !Ref JobDefinitionName, '\", ', '\"type\": \"daily\" }' ] ] - Id: !Join [ '-' [ !Ref EnvironmentName, 'DailyLambda' ] ] - Tags: - - Key: Name - Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'batchRefreshCacheEventRule' ] ] - - Key: Environment - Value: !Ref EnvironmentName - - Key: Owner - Value: !Ref Owner - - Key: Project - Value: !Ref Project + - Arn: !ImportValue 'BatchLambdaID' + Input: !Ref LambdaInputRefreshCache + Id: !Join [ '-' , [ !Ref EnvironmentName, 'DailyLambda' ] ] #Lambda permissions for events + + RefreshCachePermission: + Type: AWS::Lambda::Permission + Properties: + Action: lambda:InvokeFunction + FunctionName: !ImportValue 'BatchLambdaID' + Principal: events.amazonaws.com + SourceArn: !GetAtt RunRefreshCacheBatch.Arn + AnnualLambdaPermission: Type: AWS::Lambda::Permission Properties: Action: lambda:InvokeFunction - FunctionName: - - !ImportValue - !Sub: '${BatchLambdaStackName}-BatchLambdaID' + FunctionName: !ImportValue 'BatchLambdaID' Principal: events.amazonaws.com SourceArn: !GetAtt RunAnnualBatch.Arn - Tags: - - Key: Name - Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'batchAnnualLambdaPermission' ] ] - - Key: Environment - Value: !Ref EnvironmentName - - Key: Owner - Value: !Ref Owner - - Key: Project - Value: !Ref Project MonthlyLambdaPermission: Type: AWS::Lambda::Permission Properties: Action: lambda:InvokeFunction - FunctionName: - - !ImportValue - !Sub: '${BatchLambdaStackName}-BatchLambdaID' + FunctionName: !ImportValue 'BatchLambdaID' Principal: events.amazonaws.com SourceArn: !GetAtt RunMonthlyBatch.Arn - Tags: - - Key: Name - Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'batchMonthlyLambdaPermission' ] ] - - Key: Environment - Value: !Ref EnvironmentName - - Key: Owner - Value: !Ref Owner - - Key: Project - Value: !Ref Project WeeklyLambdaPermission: Type: AWS::Lambda::Permission Properties: Action: lambda:InvokeFunction - FunctionName: - - !ImportValue - !Sub: '${BatchLambdaStackName}-BatchLambdaID' + FunctionName: !ImportValue 'BatchLambdaID' Principal: events.amazonaws.com SourceArn: !GetAtt RunWeeklyBatch.Arn - Tags: - - Key: Name - Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'batchWeeklyLambdaPermission' ] ] - - Key: Environment - Value: !Ref EnvironmentName - - Key: Owner - Value: !Ref Owner - - Key: Project - Value: !Ref Project DailyLambdaPermission: Type: AWS::Lambda::Permission Properties: Action: lambda:InvokeFunction - FunctionName: - - !ImportValue - !Sub: '${BatchLambdaStackName}-BatchLambdaID' + FunctionName: !ImportValue 'BatchLambdaID' Principal: events.amazonaws.com SourceArn: !GetAtt RunDailyBatch.Arn - Tags: - - Key: Name - Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'batchWeeklyLambdaPermission' ] ] - - Key: Environment - Value: !Ref EnvironmentName - - Key: Owner - Value: !Ref Owner - - Key: Project - Value: !Ref Project Outputs: BatchTaskRoleARN: diff --git a/aws/cloud-formation/batchSystem/dev-batch-system-parameter.json b/aws/cloud-formation/batchSystem/dev-batch-system-parameter.json index c74f3211e4..106019a523 100644 --- a/aws/cloud-formation/batchSystem/dev-batch-system-parameter.json +++ b/aws/cloud-formation/batchSystem/dev-batch-system-parameter.json @@ -21,7 +21,7 @@ }, { "ParameterKey": "BatchLambdaStackName", - "ParameterValue": "digiroad-batch-lambda-stack" + "ParameterValue": "batch-lambda-stack" }, { "ParameterKey": "Subnet1", @@ -42,5 +42,25 @@ { "ParameterKey": "ApplicationName", "ParameterValue": "digiroad2" + }, + { + "ParameterKey": "LambdaInputAnnual", + "ParameterValue": "{ \"jobName\": \"DEV\", \"jobDefinition\": \"DEVBatchDefinition\", \"type\": \"annual\" }" + }, + { + "ParameterKey": "LambdaInputMonthly", + "ParameterValue": "{ \"jobName\": \"DEV\", \"jobDefinition\": \"DEVBatchDefinition\", \"type\": \"monthly\" }" + }, + { + "ParameterKey": "LambdaInputWeekly", + "ParameterValue": "{ \"jobName\": \"DEV\", \"jobDefinition\": \"DEVBatchDefinition\", \"type\": \"weekly\" }" + }, + { + "ParameterKey": "LambdaInputDaily", + "ParameterValue": "{ \"jobName\": \"DEV\", \"jobDefinition\": \"DEVBatchDefinition\", \"type\": \"daily\" }" + }, + { + "ParameterKey": "LambdaInputRefreshCache", + "ParameterValue": "{ \"jobName\": \"DEV\", \"jobDefinition\": \"DEVBatchDefinition\", \"type\": \"refresh_cache\" }" } ] \ No newline at end of file diff --git a/aws/cloud-formation/batchSystem/prod-batch-system-parameter.json b/aws/cloud-formation/batchSystem/prod-batch-system-parameter.json index db9d0c8687..68d2f401be 100644 --- a/aws/cloud-formation/batchSystem/prod-batch-system-parameter.json +++ b/aws/cloud-formation/batchSystem/prod-batch-system-parameter.json @@ -21,7 +21,7 @@ }, { "ParameterKey": "BatchLambdaStackName", - "ParameterValue": "digiroad-batch-lambda-stack" + "ParameterValue": "batch-lambda-stack" }, { "ParameterKey": "Subnet1", @@ -42,5 +42,25 @@ { "ParameterKey": "ApplicationName", "ParameterValue": "digiroad2" + }, + { + "ParameterKey": "LambdaInputAnnual", + "ParameterValue": "{\"jobName\": \"Prod\", \"jobDefinition\": \"ProdBatchJobDefinition\", \"type\":\"annual\"}" + }, + { + "ParameterKey": "LambdaInputMonthly", + "ParameterValue": "{\"jobName\": \"Prod\", \"jobDefinition\": \"ProdBatchJobDefinition\", \"type\":\"monthly\"}" + }, + { + "ParameterKey": "LambdaInputWeekly", + "ParameterValue": "{\"jobName\": \"Prod\", \"jobDefinition\": \"ProdBatchJobDefinition\", \"type\":\"weekly\"}" + }, + { + "ParameterKey": "LambdaInputDaily", + "ParameterValue": "{\"jobName\": \"Prod\", \"jobDefinition\": \"ProdBatchJobDefinition\", \"type\":\"daily\"}" + }, + { + "ParameterKey": "LambdaInputRefreshCache", + "ParameterValue": "{\"jobName\": \"Prod\", \"jobDefinition\": \"ProdBatchJobDefinition\", \"type\":\"refresh_cache\"}" } ] \ No newline at end of file diff --git a/aws/cloud-formation/batchSystem/qa-batch-system-parameter.json b/aws/cloud-formation/batchSystem/qa-batch-system-parameter.json index 984fd43d0c..b52bd8220e 100644 --- a/aws/cloud-formation/batchSystem/qa-batch-system-parameter.json +++ b/aws/cloud-formation/batchSystem/qa-batch-system-parameter.json @@ -21,7 +21,7 @@ }, { "ParameterKey": "BatchLambdaStackName", - "ParameterValue": "BatchLambdaStack" + "ParameterValue": "batch-lambda-stack" }, { "ParameterKey": "Subnet1", @@ -42,5 +42,25 @@ { "ParameterKey": "ApplicationName", "ParameterValue": "digiroad2" + }, + { + "ParameterKey": "LambdaInputAnnual", + "ParameterValue": "{ \"jobName\": \"QA\", \"jobDefinition\": \"QABatchDefinition\", \"type\": \"annual\" }" + }, + { + "ParameterKey": "LambdaInputMonthly", + "ParameterValue": "{ \"jobName\": \"QA\", \"jobDefinition\": \"QABatchDefinition\", \"type\": \"monthly\" }" + }, + { + "ParameterKey": "LambdaInputWeekly", + "ParameterValue": "{ \"jobName\": \"QA\", \"jobDefinition\": \"QABatchDefinition\", \"type\": \"weekly\" }" + }, + { + "ParameterKey": "LambdaInputDaily", + "ParameterValue": "{ \"jobName\": \"QA\", \"jobDefinition\": \"QABatchDefinition\", \"type\": \"daily\" }" + }, + { + "ParameterKey": "LambdaInputRefreshCache", + "ParameterValue": "{ \"jobName\": \"QA\", \"jobDefinition\": \"QABatchDefinition\", \"type\": \"refresh_cache\" }" } ] \ No newline at end of file From 19cd6d9c6b9e03908efc3008bc4520d7cce90acb Mon Sep 17 00:00:00 2001 From: "antti.ahopelto" Date: Mon, 16 May 2022 09:54:25 +0300 Subject: [PATCH 10/37] DROTH-3214 Fixed pipeline permissions --- .../cicd/dev-batchLambdaPipeline.yaml | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/aws/cloud-formation/batchSystem/batchLambda/cicd/dev-batchLambdaPipeline.yaml b/aws/cloud-formation/batchSystem/batchLambda/cicd/dev-batchLambdaPipeline.yaml index 368e25f5fe..04d1980894 100644 --- a/aws/cloud-formation/batchSystem/batchLambda/cicd/dev-batchLambdaPipeline.yaml +++ b/aws/cloud-formation/batchSystem/batchLambda/cicd/dev-batchLambdaPipeline.yaml @@ -101,6 +101,7 @@ Resources: Action: - 'lambda:*' - 'codebuild:*' + - 'codecommit:*' - 's3:*' Resource: '*' @@ -210,23 +211,7 @@ Resources: - Effect: "Allow" Resource: "*" Action: - - "ecr:GetAuthorizationToken" - - "ecr:BatchCheckLayerAvailability" - - "ecr:GetDownloadUrlForLayer" - - "ecr:GetRepositoryPolicy" - - "ecr:DescribeRepositories" - - "ecr:ListImages" - - "ecr:DescribeImages" - - "ecr:BatchGetImage" - - "ecr:GetLifecyclePolicy" - - "ecr:GetLifecyclePolicyPreview" - - "ecr:ListTagsForResource" - - "ecr:DescribeImageScanFindings" - - "ecr:InitiateLayerUpload" - - "ecr:UploadLayerPart" - - "ecr:CompleteLayerUpload" - - "ecr:PutImage" - - "ssm:GetParameters" + - "lambda:UpdateFunctionCode" - Effect: "Allow" Resource: "*" Action: From 0b6d370856de28391f4270b7cd1d9f60d9b8f84b Mon Sep 17 00:00:00 2001 From: "antti.ahopelto" Date: Mon, 16 May 2022 12:16:13 +0300 Subject: [PATCH 11/37] DROTH-3214 New function name for lambda --- aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml | 2 +- .../batchSystem/batchLambda/cicd/dev-batchLambdaPipeline.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml b/aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml index d79758655e..ac53186a1e 100644 --- a/aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml +++ b/aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml @@ -35,7 +35,7 @@ Resources: Value: !Ref Owner - Key: Project Value: !Ref Project - FunctionName: "Dev-Batch-Add-Jobs-To-Queue" + FunctionName: "Batch-Add-Jobs-To-Queue-New" Handler: AddJobToQueue.handler Role: !GetAtt BatchLambdaRole.Arn Runtime: nodejs14.x diff --git a/aws/cloud-formation/batchSystem/batchLambda/cicd/dev-batchLambdaPipeline.yaml b/aws/cloud-formation/batchSystem/batchLambda/cicd/dev-batchLambdaPipeline.yaml index 04d1980894..6e8c97c279 100644 --- a/aws/cloud-formation/batchSystem/batchLambda/cicd/dev-batchLambdaPipeline.yaml +++ b/aws/cloud-formation/batchSystem/batchLambda/cicd/dev-batchLambdaPipeline.yaml @@ -190,7 +190,7 @@ Resources: post_build: commands: - echo Updating Dev account lambda code - - aws lambda update-function-code --function-name Batch-Add-Jobs-To-Queue --zip-file fileb://deployment_package.zip + - aws lambda update-function-code --function-name Batch-Add-Jobs-To-Queue-New --zip-file fileb://deployment_package.zip CodeBuildRole: Type: AWS::IAM::Role From 331bb14cd73b231021a92581497035cd85dc6959 Mon Sep 17 00:00:00 2001 From: "antti.ahopelto" Date: Mon, 16 May 2022 13:08:08 +0300 Subject: [PATCH 12/37] DROTH-3214 fix prod pipeline YAML, add artifact store --- .../cicd/prod-batchLambdaPipeline.yaml | 101 ++++++++++++------ .../batchSystem/batchSystem.yaml | 2 +- 2 files changed, 70 insertions(+), 33 deletions(-) diff --git a/aws/cloud-formation/batchSystem/batchLambda/cicd/prod-batchLambdaPipeline.yaml b/aws/cloud-formation/batchSystem/batchLambda/cicd/prod-batchLambdaPipeline.yaml index 5446c6ea26..e7ffc50fe2 100644 --- a/aws/cloud-formation/batchSystem/batchLambda/cicd/prod-batchLambdaPipeline.yaml +++ b/aws/cloud-formation/batchSystem/batchLambda/cicd/prod-batchLambdaPipeline.yaml @@ -1,7 +1,7 @@ AWSTemplateFormatVersion: "2010-09-09" Parameters: DevBucketName: - Decription: Name of dev bucket for lambda deployment + Description: Name of dev bucket for lambda deployment Type: String LambdaCodeS3Key: @@ -13,51 +13,88 @@ Parameters: Type: String Resources: + LambdaArtifactStore: + Type: AWS::S3::Bucket + Properties: + BucketName: 'prod-batch-lambda-build-artifact-store' + BatchLambdaPipeline: Type: AWS::CodePipeline::Pipeline Properties: Name: 'prod-batch-lambda-pipeline' RoleArn: !GetAtt PipelineRole.Arn - RestartExecutionOnUpdate: true Stages: - - Name: Source + - + Name: Source Actions: - Name: SourceAction - ActionTypeID: - Category: Source - Owner: AWS - Provider: S3 - Version: 1 - OutputArtifacts: - - Name: SourceOutput - Configuration: - S3Bucket: !Ref DevBucket - S3ObjectKey: !Ref LambdaCodeS3Key - PollForSourceChanges: 'true' - RunOrder: 1 - - Name: Accept + - + Name: SourceAction + ActionTypeId: + Category: Source + Owner: AWS + Provider: S3 + Version: 1 + OutputArtifacts: + - + Name: SourceOutput + Configuration: + S3Bucket: !Ref DevBucketName + S3ObjectKey: !Ref LambdaCodeS3Key + PollForSourceChanges: 'true' + RunOrder: 1 + - + Name: Accept Actions: - - Name: Approve_Lambda_zip_For_Production + - + Name: Approve_Lambda_zip_For_Production ActionTypeId: Category: Approval Owner: AWS Version: 1 Provider: Manual - - Name: Build + - + Name: Build Actions: - Name: BuildAction - InputArtifacts: - - Name: SourceOutput - ActionTypeId: - Category: Build - Owner: AWS - Version: 1 - Provider: CodeBuild - OutputArtifacts: - - Name: Built - Configuration: - ProjectName: !Ref CodeBuild - RunOrder: 1 + - + Name: BuildAction + InputArtifacts: + - Name: SourceOutput + ActionTypeId: + Category: Build + Owner: AWS + Version: 1 + Provider: CodeBuild + OutputArtifacts: + - Name: Built + Configuration: + ProjectName: !Ref CodeBuild + RunOrder: 1 + ArtifactStore: + Type: S3 + Location: !Ref LambdaArtifactStore + + PipelineRole: + Type: 'AWS::IAM::Role' + Properties: + RoleName: 'prod-batch-lambda-pipeline-role' + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - codepipeline.amazonaws.com + Action: 'sts:AssumeRole' + Path: / + Policies: + - PolicyName: 'prod-batch-lambda-pipeline-policy' + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - 's3:*' + Resource: '*' CodeBuild: Type: AWS::CodeBuild::Project diff --git a/aws/cloud-formation/batchSystem/batchSystem.yaml b/aws/cloud-formation/batchSystem/batchSystem.yaml index b3458834be..61f5476456 100644 --- a/aws/cloud-formation/batchSystem/batchSystem.yaml +++ b/aws/cloud-formation/batchSystem/batchSystem.yaml @@ -86,7 +86,7 @@ Resources: - Fn::ImportValue: !Ref Subnet2 SecurityGroupIds: - - 'sg-011055fc8eac78b1d' + - !Ref BatchSecurityGroup ServiceRole: Ref: BatchServiceRole Tags: {"Name" : "Digiroad2-BatchComputeEnvironment", "Environment" : !Ref EnvironmentName, "Owner": !Ref Owner, "Project": !Ref Project} From dd38ad3261394adbecf64c72bdb017a6ba9b6c41 Mon Sep 17 00:00:00 2001 From: "antti.ahopelto" Date: Mon, 16 May 2022 14:46:55 +0300 Subject: [PATCH 13/37] DROTH-3214 add parameters to prod deployment bucket, fix YAML --- .../prod-deployment-bucket-parameter.json | 22 +++++++++ .../cicd/prodBatchLambdaDeploymentBucket.yaml | 47 ++++++++++++++----- 2 files changed, 58 insertions(+), 11 deletions(-) create mode 100644 aws/cloud-formation/batchSystem/batchLambda/cicd/prod-deployment-bucket-parameter.json diff --git a/aws/cloud-formation/batchSystem/batchLambda/cicd/prod-deployment-bucket-parameter.json b/aws/cloud-formation/batchSystem/batchLambda/cicd/prod-deployment-bucket-parameter.json new file mode 100644 index 0000000000..a72271f4d6 --- /dev/null +++ b/aws/cloud-formation/batchSystem/batchLambda/cicd/prod-deployment-bucket-parameter.json @@ -0,0 +1,22 @@ +[ + { + "ParameterKey": "BucketName", + "ParameterValue": "prod-batch-lambda-deployment-bucket" + }, + { + "ParameterKey": "DevAccountID", + "ParameterValue": "475079312496" + }, + { + "ParameterKey": "Owner", + "ParameterValue": "vayla" + }, + { + "ParameterKey": "Project", + "ParameterValue": "digiroad2" + }, + { + "ParameterKey": "ApplicationName", + "ParameterValue": "digiroad2" + } +] \ No newline at end of file diff --git a/aws/cloud-formation/batchSystem/batchLambda/cicd/prodBatchLambdaDeploymentBucket.yaml b/aws/cloud-formation/batchSystem/batchLambda/cicd/prodBatchLambdaDeploymentBucket.yaml index 72630d0409..4db8305bf9 100644 --- a/aws/cloud-formation/batchSystem/batchLambda/cicd/prodBatchLambdaDeploymentBucket.yaml +++ b/aws/cloud-formation/batchSystem/batchLambda/cicd/prodBatchLambdaDeploymentBucket.yaml @@ -1,27 +1,52 @@ AWSTemplateFormatVersion: "2010-09-09" +Parameters: + BucketName: + Description: "Name for prod lambda deployment bucket" + Type: String + DevAccountID: + Description: "ID of Development AWS account" + Type: String + Owner: + Description: "Owner of the of application, that these resources are created for. Used when tagging the resources" + Type: String + Project: + Description: "Project of the of application, that these resources are created for. Used when tagging the resources" + Type: String + ApplicationName: + Description: "Name of the application (no whitespace or special characters)" + Type: String + Resources: LambdaDeploymentBucket: Type: AWS::S3::Bucket Properties: - BucketName: 'prod-batch-lambda-deployment-bucket' - AccessControl: 'private' + BucketName: !Ref BucketName + AccessControl: 'Private' + Tags: + - Key: Name + Value: !Join [ '-', [!Ref ApplicationName, 'lambdaDeploymentBucket' ] ] + - Key: Owner + Value: !Ref Owner + - Key: Project + Value: !Ref Project DeploymentBucketPolicy: Type: AWS::S3::BucketPolicy Properties: Bucket: !Ref LambdaDeploymentBucket PolicyDocument: + Version: 2012-10-17 Statement: - Sid: "PolicyForAllowUploadWithACL" - Effect: Allow - Principal: - AWS: '475079312496' - Action: "s3:PutObject" - Resource: "arn:aws:s3:::prod-batch-lambda-deployment-bucket/*" - Condition: { - "StringEquals": {"s3:x-amz-acl":"bucket-owner-full-control"} - } + - Action: + - 's3:PutObject' + Effect: Allow + Principal: + AWS: !Ref DevAccountID + Resource: "arn:aws:s3:::prod-batch-lambda-deployment-bucket/*" + Condition: { + "StringEquals": {"s3:x-amz-acl":"bucket-owner-full-control"} + } Outputs: BucketOutput: From 0468daa7c5aa1c5bf936120686965b4cdc0633b3 Mon Sep 17 00:00:00 2001 From: "antti.ahopelto" Date: Tue, 17 May 2022 08:48:11 +0300 Subject: [PATCH 14/37] DROTH-3214 added --parameters to S3 creation guide, renamed parameter --- .../batchLambda/cicd/prod-deployment-bucket-parameter.json | 2 +- .../batchLambda/cicd/prodBatchLambdaDeploymentBucket.yaml | 4 ++-- aws/cloud-formation/fargateService/prod/README.md | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/aws/cloud-formation/batchSystem/batchLambda/cicd/prod-deployment-bucket-parameter.json b/aws/cloud-formation/batchSystem/batchLambda/cicd/prod-deployment-bucket-parameter.json index a72271f4d6..1e6705c407 100644 --- a/aws/cloud-formation/batchSystem/batchLambda/cicd/prod-deployment-bucket-parameter.json +++ b/aws/cloud-formation/batchSystem/batchLambda/cicd/prod-deployment-bucket-parameter.json @@ -4,7 +4,7 @@ "ParameterValue": "prod-batch-lambda-deployment-bucket" }, { - "ParameterKey": "DevAccountID", + "ParameterKey": "AccountID", "ParameterValue": "475079312496" }, { diff --git a/aws/cloud-formation/batchSystem/batchLambda/cicd/prodBatchLambdaDeploymentBucket.yaml b/aws/cloud-formation/batchSystem/batchLambda/cicd/prodBatchLambdaDeploymentBucket.yaml index 4db8305bf9..615d42d259 100644 --- a/aws/cloud-formation/batchSystem/batchLambda/cicd/prodBatchLambdaDeploymentBucket.yaml +++ b/aws/cloud-formation/batchSystem/batchLambda/cicd/prodBatchLambdaDeploymentBucket.yaml @@ -4,7 +4,7 @@ Parameters: BucketName: Description: "Name for prod lambda deployment bucket" Type: String - DevAccountID: + AccountID: Description: "ID of Development AWS account" Type: String Owner: @@ -42,7 +42,7 @@ Resources: - 's3:PutObject' Effect: Allow Principal: - AWS: !Ref DevAccountID + AWS: !Ref AccountID Resource: "arn:aws:s3:::prod-batch-lambda-deployment-bucket/*" Condition: { "StringEquals": {"s3:x-amz-acl":"bucket-owner-full-control"} diff --git a/aws/cloud-formation/fargateService/prod/README.md b/aws/cloud-formation/fargateService/prod/README.md index fb0e1613bd..19919d92dd 100644 --- a/aws/cloud-formation/fargateService/prod/README.md +++ b/aws/cloud-formation/fargateService/prod/README.md @@ -95,7 +95,8 @@ aws cloudformation create-stack \ ``` aws cloudformation create-stack \ --stack-name [esim. digiroad-batch-lambda-bucket] \ ---template-body file://aws/cloud-formation/batchSystem/batchLambda/cicd/prodBatchLambdaDeploymentBucket.yaml +--template-body file://aws/cloud-formation/batchSystem/batchLambda/cicd/prodBatchLambdaDeploymentBucket.yaml \ +--parameters ``` S3-Bucketin luonnin jälkeen pyydä kehitystiimiä toimittamaan lambdan koodi .zip tiedostona sinne From 4671d28466e502cef153aab759305d1733defde2 Mon Sep 17 00:00:00 2001 From: "antti.ahopelto" Date: Tue, 17 May 2022 08:49:18 +0300 Subject: [PATCH 15/37] DROTH-3214 fix params --- aws/cloud-formation/fargateService/prod/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/cloud-formation/fargateService/prod/README.md b/aws/cloud-formation/fargateService/prod/README.md index 19919d92dd..c0612c683a 100644 --- a/aws/cloud-formation/fargateService/prod/README.md +++ b/aws/cloud-formation/fargateService/prod/README.md @@ -96,7 +96,7 @@ aws cloudformation create-stack \ aws cloudformation create-stack \ --stack-name [esim. digiroad-batch-lambda-bucket] \ --template-body file://aws/cloud-formation/batchSystem/batchLambda/cicd/prodBatchLambdaDeploymentBucket.yaml \ ---parameters +--parameters file://aws/cloud-formation/batchSystem/batchLambda/cicd/prod-deployment-bucket-parameter.json ``` S3-Bucketin luonnin jälkeen pyydä kehitystiimiä toimittamaan lambdan koodi .zip tiedostona sinne From 1839b583430ecacf30245a0f189f45f3afe0a582 Mon Sep 17 00:00:00 2001 From: "antti.ahopelto" Date: Tue, 17 May 2022 13:02:46 +0300 Subject: [PATCH 16/37] DROTH-3259 enable caching QA job definition --- aws/cloud-formation/batchSystem/QAbatchJobDefinition.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/cloud-formation/batchSystem/QAbatchJobDefinition.json b/aws/cloud-formation/batchSystem/QAbatchJobDefinition.json index fbc8ac3bf7..f56f0a6e58 100644 --- a/aws/cloud-formation/batchSystem/QAbatchJobDefinition.json +++ b/aws/cloud-formation/batchSystem/QAbatchJobDefinition.json @@ -106,7 +106,7 @@ }, { "name": "caching", - "value": "false" + "value": "true" }, { "name": "cacheHostname", From fc6933a930fb5500ee6e7e1095dc0cc84aaec872 Mon Sep 17 00:00:00 2001 From: "antti.ahopelto" Date: Tue, 17 May 2022 13:03:46 +0300 Subject: [PATCH 17/37] DROTH-3259 also enable caching prod job definition --- aws/cloud-formation/batchSystem/ProdBatchJobDefinition.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/cloud-formation/batchSystem/ProdBatchJobDefinition.json b/aws/cloud-formation/batchSystem/ProdBatchJobDefinition.json index 78a16d7604..6f05a0c7a5 100644 --- a/aws/cloud-formation/batchSystem/ProdBatchJobDefinition.json +++ b/aws/cloud-formation/batchSystem/ProdBatchJobDefinition.json @@ -106,7 +106,7 @@ }, { "name": "caching", - "value": "false" + "value": "true" }, { "name": "cacheHostname", From 605662831032eab547745a8fae2303e97faf21f8 Mon Sep 17 00:00:00 2001 From: "antti.ahopelto" Date: Tue, 17 May 2022 14:48:48 +0300 Subject: [PATCH 18/37] DROTH-3198 Out of memory fix --- .../liikennevirasto/digiroad2/util/RefreshRoadLinkCache.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/digiroad2-oracle/src/main/scala/fi/liikennevirasto/digiroad2/util/RefreshRoadLinkCache.scala b/digiroad2-oracle/src/main/scala/fi/liikennevirasto/digiroad2/util/RefreshRoadLinkCache.scala index 56f8cc599a..45c7e38a0c 100644 --- a/digiroad2-oracle/src/main/scala/fi/liikennevirasto/digiroad2/util/RefreshRoadLinkCache.scala +++ b/digiroad2-oracle/src/main/scala/fi/liikennevirasto/digiroad2/util/RefreshRoadLinkCache.scala @@ -30,10 +30,10 @@ object RefreshRoadLinkCache { } if (flushSuccess) { - val roadLinks = municipalities.flatMap(municipality => { + municipalities.foreach(municipality => { roadLinkService.getRoadLinksAndComplementaryLinksFromVVHByMunicipality(municipality) }) - logger.info("Cached " + roadLinks.size + " roadlinks with overrided properties from database") + logger.info("Cached roadlinks with overrided properties from database") } else logger.error("Flushing cache failed") } From dfc4c2b02439198ff5c0057c80f942e8f81e90e7 Mon Sep 17 00:00:00 2001 From: "antti.ahopelto" Date: Wed, 18 May 2022 13:17:11 +0300 Subject: [PATCH 19/37] DROTH-3198 Fixed roleArn in job definitions to correct new arn. Fixed errors found when creating stack on prod account --- aws/cloud-formation/batchSystem/DEVbatchJobDefinition.json | 2 +- aws/cloud-formation/batchSystem/ProdBatchJobDefinition.json | 2 +- aws/cloud-formation/batchSystem/QAbatchJobDefinition.json | 2 +- aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml | 2 +- .../batchSystem/batchLambda/prod-batch-lambda-parameter.json | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/aws/cloud-formation/batchSystem/DEVbatchJobDefinition.json b/aws/cloud-formation/batchSystem/DEVbatchJobDefinition.json index 0c7830cf49..ed906bd09a 100644 --- a/aws/cloud-formation/batchSystem/DEVbatchJobDefinition.json +++ b/aws/cloud-formation/batchSystem/DEVbatchJobDefinition.json @@ -22,7 +22,7 @@ "value": "4" } ], - "executionRoleArn": "arn:aws:iam::475079312496:role/batchSystem-BatchTaskRole-GSUD7E3H8KV4", + "executionRoleArn": "arn:aws:iam::475079312496:role/DEV-batchSystem-BatchTaskRole-VGL42N9TANTI", "environment": [ { "name": "containerCPU", diff --git a/aws/cloud-formation/batchSystem/ProdBatchJobDefinition.json b/aws/cloud-formation/batchSystem/ProdBatchJobDefinition.json index 78a16d7604..28f526df52 100644 --- a/aws/cloud-formation/batchSystem/ProdBatchJobDefinition.json +++ b/aws/cloud-formation/batchSystem/ProdBatchJobDefinition.json @@ -22,7 +22,7 @@ "value": "4" } ], - "executionRoleArn": "arn:aws:iam::475079312496:role/batchSystem-BatchTaskRole-GSUD7E3H8KV4", + "executionRoleArn": "arn:aws:iam::920408837790:role/digiroad-batch-system-BatchTaskRole-NTGCBVRU7CCR", "environment": [ { "name": "containerCPU", diff --git a/aws/cloud-formation/batchSystem/QAbatchJobDefinition.json b/aws/cloud-formation/batchSystem/QAbatchJobDefinition.json index fbc8ac3bf7..c6ef7c4c89 100644 --- a/aws/cloud-formation/batchSystem/QAbatchJobDefinition.json +++ b/aws/cloud-formation/batchSystem/QAbatchJobDefinition.json @@ -22,7 +22,7 @@ "value": "4" } ], - "executionRoleArn": "arn:aws:iam::475079312496:role/batchSystem-BatchTaskRole-GSUD7E3H8KV4", + "executionRoleArn": "arn:aws:iam::475079312496:role/QA-batchSystem-BatchTaskRole-1QO0VDAJOYS45", "environment": [ { "name": "containerCPU", diff --git a/aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml b/aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml index ac53186a1e..8885f34073 100644 --- a/aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml +++ b/aws/cloud-formation/batchSystem/batchLambda/batchLambda.yaml @@ -24,7 +24,7 @@ Resources: Type: AWS::Lambda::Function Properties: Code: - S3Bucket: !ImportValue 'batch-lambda-deployment-bucket' + S3Bucket: !Ref BucketName S3Key: !Ref S3ObjectKey Tags: - Key: Name diff --git a/aws/cloud-formation/batchSystem/batchLambda/prod-batch-lambda-parameter.json b/aws/cloud-formation/batchSystem/batchLambda/prod-batch-lambda-parameter.json index 3d4f1f370a..bb995914c7 100644 --- a/aws/cloud-formation/batchSystem/batchLambda/prod-batch-lambda-parameter.json +++ b/aws/cloud-formation/batchSystem/batchLambda/prod-batch-lambda-parameter.json @@ -1,11 +1,11 @@ [ { "ParameterKey": "BucketName", - "ParameterValue": "dev-batch-lambda-deployment-bucket" + "ParameterValue": "prod-batch-lambda-deployment-bucket" }, { "ParameterKey": "S3ObjectKey", - "ParameterValue": "batch-lambda-deployment" + "ParameterValue": "deployment_package.zip" }, { "ParameterKey": "EnvironmentName", From 9cb33da8d6a1b0097d0c2ec32e85acad3de50ab1 Mon Sep 17 00:00:00 2001 From: "antti.ahopelto" Date: Wed, 18 May 2022 13:18:26 +0300 Subject: [PATCH 20/37] DROTH-3198 included fix in batchSystem.yaml --- aws/cloud-formation/batchSystem/batchSystem.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/cloud-formation/batchSystem/batchSystem.yaml b/aws/cloud-formation/batchSystem/batchSystem.yaml index 61f5476456..37d4fbe23e 100644 --- a/aws/cloud-formation/batchSystem/batchSystem.yaml +++ b/aws/cloud-formation/batchSystem/batchSystem.yaml @@ -225,7 +225,7 @@ Resources: Name: !Join [ '-', [ !Ref EnvironmentName, 'batch-event-notify' ]] State: "ENABLED" Targets: - - Arn: !Join ['', ["arn:aws:sns:eu-west-1:475079312496:", !Ref EnvironmentName, '-', !Ref SNSTopicName]] + - Arn: !Join ['', ["arn:aws:sns:eu-west-1:", !Ref AWS::AccountId, ":", !Ref EnvironmentName, '-', !Ref SNSTopicName]] Id: "123321" InputTransformer: InputTemplate: | From 771945885b3b155d7b35de5581cf281b04ccd351 Mon Sep 17 00:00:00 2001 From: "antti.ahopelto" Date: Wed, 18 May 2022 15:28:38 +0300 Subject: [PATCH 21/37] DROTH-3198 added exit() for refreshCache --- .../scala/fi/liikennevirasto/digiroad2/util/DataFixture.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/digiroad2-oracle/src/main/scala/fi/liikennevirasto/digiroad2/util/DataFixture.scala b/digiroad2-oracle/src/main/scala/fi/liikennevirasto/digiroad2/util/DataFixture.scala index bc0b4606d0..de85ab69d9 100644 --- a/digiroad2-oracle/src/main/scala/fi/liikennevirasto/digiroad2/util/DataFixture.scala +++ b/digiroad2-oracle/src/main/scala/fi/liikennevirasto/digiroad2/util/DataFixture.scala @@ -2339,6 +2339,7 @@ object DataFixture { UpdateIncompleteLinkList.runUpdate() case Some("refresh_road_link_cache") => RefreshRoadLinkCache.refreshCache() + exit() //For a currently unknown reason refreshCache batch doesn't exit automatically upon completion case _ => println("Usage: DataFixture test | import_roadlink_data |" + " split_speedlimitchains | split_linear_asset_chains | dropped_assets_csv | dropped_manoeuvres_csv |" + " unfloat_linear_assets | expire_split_assets_without_mml | generate_values_for_lit_roads | get_addresses_to_masstransitstops_from_vvh |" + From cac7d2b0fe43f4b7731d8b2b4bf6c82fb21d6d69 Mon Sep 17 00:00:00 2001 From: sasuolander Date: Wed, 18 May 2022 16:16:09 +0300 Subject: [PATCH 22/37] DROTH-3258 add new endpoint and needed properties --- .../digiroad2/client/vvh/VVHClient.scala | 34 ++++++++++++++++++- .../digiroad2/util/Digiroad2Properties.scala | 10 ++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/digiroad2-oracle/src/main/scala/fi/liikennevirasto/digiroad2/client/vvh/VVHClient.scala b/digiroad2-oracle/src/main/scala/fi/liikennevirasto/digiroad2/client/vvh/VVHClient.scala index 550947bf43..f41c08070f 100644 --- a/digiroad2-oracle/src/main/scala/fi/liikennevirasto/digiroad2/client/vvh/VVHClient.scala +++ b/digiroad2-oracle/src/main/scala/fi/liikennevirasto/digiroad2/client/vvh/VVHClient.scala @@ -6,7 +6,8 @@ import com.vividsolutions.jts.geom.Polygon import fi.liikennevirasto.digiroad2.Point import fi.liikennevirasto.digiroad2.asset._ import fi.liikennevirasto.digiroad2.linearasset.RoadLinkLike -import fi.liikennevirasto.digiroad2.util.LogUtils +import fi.liikennevirasto.digiroad2.util.{Digiroad2Properties, LogUtils, OAGAuthPropertyReader} +import org.apache.commons.codec.binary.Base64 import org.apache.http.NameValuePair import org.apache.http.client.entity.UrlEncodedFormEntity import org.apache.http.client.methods.{HttpGet, HttpPost} @@ -245,6 +246,27 @@ trait VVHClientOperations { lazy val logger = LoggerFactory.getLogger(getClass) + + class VVHAuthPropertyReader { + private def getUsername: String = { + val loadedKeyString = Digiroad2Properties.vvhRestUsername + if (loadedKeyString == null) + throw new IllegalArgumentException("Missing OAG username") + loadedKeyString + } + + private def getPassword: String = { + val loadedKeyString = Digiroad2Properties.vvhRestPassword + if (loadedKeyString == null) + throw new IllegalArgumentException("Missing OAG Password") + loadedKeyString + } + + def getAuthInBase64: String = { + Base64.encodeBase64String((getUsername + ":" + getPassword).getBytes) + } + } + protected def anyToDouble(number: Any): Option[Double] = number match { case bi: BigInt => Some(bi.toDouble) case i: Int => Some(i.toDouble) @@ -373,6 +395,9 @@ trait VVHClientOperations { val fetchVVHStartTime = System.currentTimeMillis() val request = new HttpGet(url) val client = HttpClientBuilder.create().build() + val vVHAuthPropertyReader = new VVHAuthPropertyReader + request.addHeader("Authorization", "Basic " + vVHAuthPropertyReader.getAuthInBase64) + val response = client.execute(request) try { mapFields(parse(StreamInput(response.getEntity.getContent)).values.asInstanceOf[Map[String, Any]], url) @@ -389,6 +414,10 @@ trait VVHClientOperations { val request = new HttpPost(url) request.setEntity(new UrlEncodedFormEntity(formparams, "utf-8")) val client = HttpClientBuilder.create().build() + + val vVHAuthPropertyReader = new VVHAuthPropertyReader + request.addHeader("Authorization", "Basic " + vVHAuthPropertyReader.getAuthInBase64) + val response = client.execute(request) try { mapFields(parse(StreamInput(response.getEntity.getContent)).values.asInstanceOf[Map[String, Any]], url) @@ -1118,6 +1147,9 @@ class VVHComplementaryClient(vvhRestApiEndPoint: String) extends VVHRoadLinkClie val request = new HttpPost(url) request.setEntity(new UrlEncodedFormEntity(createFormParams(complementaryFeatures), "utf-8")) val client = HttpClientBuilder.create().build() + val vVHAuthPropertyReader = new VVHAuthPropertyReader + request.addHeader("Authorization", "Basic " + vVHAuthPropertyReader.getAuthInBase64) + val response = client.execute(request) try { val content: Map[String, Seq[Map[String, Any]]] = parse(StreamInput(response.getEntity.getContent)).values.asInstanceOf[Map[String, Seq[Map[String, Any]]]] diff --git a/digiroad2-oracle/src/main/scala/fi/liikennevirasto/digiroad2/util/Digiroad2Properties.scala b/digiroad2-oracle/src/main/scala/fi/liikennevirasto/digiroad2/util/Digiroad2Properties.scala index 08aab4fc26..e959faad02 100644 --- a/digiroad2-oracle/src/main/scala/fi/liikennevirasto/digiroad2/util/Digiroad2Properties.scala +++ b/digiroad2-oracle/src/main/scala/fi/liikennevirasto/digiroad2/util/Digiroad2Properties.scala @@ -13,6 +13,8 @@ trait Digiroad2Properties { val vvhServiceHost: String val vvhRestApiEndPoint: String val vvhRoadlinkFrozen: Boolean + val vvhRestUsername: String + val vvhRestPassword: String val viiteRestApiEndPoint: String val vkmUrl: String val vkmApiKey: String @@ -64,6 +66,7 @@ class Digiroad2PropertiesFromEnv extends Digiroad2Properties { val vvhServiceHost: String = scala.util.Properties.envOrElse("vvhServiceHost", null) val vvhRestApiEndPoint: String = scala.util.Properties.envOrElse("vvhRestApiEndPoint", null) val vvhRoadlinkFrozen: Boolean = scala.util.Properties.envOrElse("vvhRoadlink.frozen", "false").toBoolean + val viiteRestApiEndPoint: String = scala.util.Properties.envOrElse("viiteRestApiEndPoint", null) val viiteApiKey: String = scala.util.Properties.envOrElse("viite.apikey", null) val sesUsername: String = scala.util.Properties.envOrElse("ses.username", null) @@ -107,6 +110,9 @@ class Digiroad2PropertiesFromEnv extends Digiroad2Properties { codebuildVersion } } + + val vvhRestUsername: String = selectEnvType(scala.util.Properties.envOrElse("vvhRest_username", null), scala.util.Properties.envOrElse("vvhRest.username", null)) + val vvhRestPassword: String = selectEnvType(scala.util.Properties.envOrElse("vvhRest_password", null), scala.util.Properties.envOrElse("vvhRest.password", null)) val googleMapApiClientId: String = selectEnvType(scala.util.Properties.envOrElse("googlemapapi_client_id", null), scala.util.Properties.envOrElse("googlemapapi.client_id", null)) val googleMapApiCryptoKey: String = selectEnvType(scala.util.Properties.envOrElse("googlemapapi_crypto_key", null), scala.util.Properties.envOrElse("googlemapapi.crypto_key", null)) val bonecpJdbcUrl: String = selectEnvType(scala.util.Properties.envOrElse("bonecp_jdbcUrl", null), scala.util.Properties.envOrElse("bonecp.jdbcUrl", null)) @@ -142,6 +148,8 @@ class Digiroad2PropertiesFromFile extends Digiroad2Properties { override val useVVHGeometry: String = envProps.getProperty("useVVHGeometry") override val vvhServiceHost: String = envProps.getProperty("vvhServiceHost") override val vvhRestApiEndPoint: String = envProps.getProperty("vvhRestApiEndPoint") + override val vvhRestUsername: String = envOrProperties("vvhRest.username") + override val vvhRestPassword: String = envOrProperties("vvhRest.password") override val vvhRoadlinkFrozen: Boolean = envProps.getProperty("vvhRoadlink.frozen", "false").toBoolean override val viiteRestApiEndPoint: String = envOrProperties("viiteRestApiEndPoint") override val vkmUrl: String = envProps.getProperty("vkmUrl") @@ -225,6 +233,8 @@ object Digiroad2Properties { lazy val vvhServiceHost: String = properties.vvhServiceHost lazy val vvhRestApiEndPoint: String = properties.vvhRestApiEndPoint lazy val vvhRoadlinkFrozen: Boolean = properties.vvhRoadlinkFrozen + lazy val vvhRestUsername: String = properties.vvhRestUsername + lazy val vvhRestPassword: String = properties.vvhRestPassword lazy val viiteRestApiEndPoint: String = properties.viiteRestApiEndPoint lazy val vkmUrl: String = properties.vkmUrl lazy val vkmApiKey: String = properties.vkmApiKey From 1db791b87752433ae43ff53882a6d2a8a5703f36 Mon Sep 17 00:00:00 2001 From: sasuolander Date: Wed, 18 May 2022 16:18:34 +0300 Subject: [PATCH 23/37] DROTH-3258 update cloudformations --- .../batchSystem/DEVbatchJobDefinition.json | 12 ++++++++++-- .../batchSystem/ProdBatchJobDefinition.json | 13 +++++++++++-- .../batchSystem/QAbatchJobDefinition.json | 12 ++++++++++-- aws/cloud-formation/cicd/dev/cicd-github.yaml | 7 ++++++- .../parameter-store/digiroad2-parameter-store.yaml | 7 +++++++ .../taskdefinition/dev-create-taskdefinition.yaml | 8 ++++++-- .../taskdefinition/prod-create-taskdefinition.yaml | 8 ++++++-- .../taskdefinition/qa-create-taskdefinition.yaml | 8 ++++++-- 8 files changed, 62 insertions(+), 13 deletions(-) diff --git a/aws/cloud-formation/batchSystem/DEVbatchJobDefinition.json b/aws/cloud-formation/batchSystem/DEVbatchJobDefinition.json index ed906bd09a..bc2203a42e 100644 --- a/aws/cloud-formation/batchSystem/DEVbatchJobDefinition.json +++ b/aws/cloud-formation/batchSystem/DEVbatchJobDefinition.json @@ -50,11 +50,15 @@ }, { "name": "vvhServiceHost", - "value": "haproxy.vayla.fi" + "value": "api.vayla.fi" }, { "name": "vvhRestApiEndPoint", - "value": "https://haproxy.vayla.fi:2027/vvhdata/" + "value": "https://api.vayla.fi/vvhdata/" + }, + { + "name": "vvhRest.username", + "value": "svc_vvh_digiroad" }, { "name": "vvhRoadlink.frozen", @@ -132,6 +136,10 @@ "name": "bonecp.password", "valueFrom": "arn:aws:ssm:eu-west-1:475079312496:parameter/dev/bonecp.password" }, + { + "name": "vvhRest.password", + "valueFrom": "arn:aws:ssm:eu-west-1:475079312496:parameter/dev/vvhRest.password" + }, { "name": "oag.password", "valueFrom": "arn:aws:ssm:eu-west-1:475079312496:parameter/dev/authentication.oag.basic.password" diff --git a/aws/cloud-formation/batchSystem/ProdBatchJobDefinition.json b/aws/cloud-formation/batchSystem/ProdBatchJobDefinition.json index 3ef56a5ec0..f36ba9c643 100644 --- a/aws/cloud-formation/batchSystem/ProdBatchJobDefinition.json +++ b/aws/cloud-formation/batchSystem/ProdBatchJobDefinition.json @@ -50,11 +50,15 @@ }, { "name": "vvhServiceHost", - "value": "haproxy.vayla.fi" + "value": "api.vayla.fi" }, { "name": "vvhRestApiEndPoint", - "value": "https://haproxy.vayla.fi:2027/vvhdata/" + "value": "https://api.vayla.fi/vvhdata/" + }, + { + "name": "vvhRest.username", + "value": "svc_vvh_digiroad" }, { "name": "vvhRoadlink.frozen", @@ -132,10 +136,15 @@ "name": "bonecp.password", "valueFrom": "arn:aws:ssm:eu-west-1:475079312496:parameter/prod/bonecp.password" }, + { + "name": "vvhRest.password", + "valueFrom": "arn:aws:ssm:eu-west-1:475079312496:parameter/prod/vvhRest.password" + }, { "name": "oag.password", "valueFrom": "arn:aws:ssm:eu-west-1:475079312496:parameter/prod/authentication.oag.basic.password" }, + { "name": "viite.apikey", "valueFrom": "arn:aws:ssm:eu-west-1:475079312496:parameter/prod/apikey/viite" diff --git a/aws/cloud-formation/batchSystem/QAbatchJobDefinition.json b/aws/cloud-formation/batchSystem/QAbatchJobDefinition.json index 4912958c4e..81225031a6 100644 --- a/aws/cloud-formation/batchSystem/QAbatchJobDefinition.json +++ b/aws/cloud-formation/batchSystem/QAbatchJobDefinition.json @@ -50,11 +50,15 @@ }, { "name": "vvhServiceHost", - "value": "haproxy.vayla.fi" + "value": "api.vayla.fi" }, { "name": "vvhRestApiEndPoint", - "value": "https://haproxy.vayla.fi:2027/vvhdata/" + "value": "https://api.vayla.fi/vvhdata/" + }, + { + "name": "vvhRest.username", + "value": "svc_vvh_digiroad" }, { "name": "vvhRoadlink.frozen", @@ -132,6 +136,10 @@ "name": "bonecp.password", "valueFrom": "arn:aws:ssm:eu-west-1:475079312496:parameter/qa/bonecp.password" }, + { + "name": "vvhRest.password", + "valueFrom": "arn:aws:ssm:eu-west-1:475079312496:parameter/qa/vvhRest.password" + }, { "name": "oag.password", "valueFrom": "arn:aws:ssm:eu-west-1:475079312496:parameter/qa/authentication.oag.basic.password" diff --git a/aws/cloud-formation/cicd/dev/cicd-github.yaml b/aws/cloud-formation/cicd/dev/cicd-github.yaml index 7cb3625799..a3b9ab8c00 100644 --- a/aws/cloud-formation/cicd/dev/cicd-github.yaml +++ b/aws/cloud-formation/cicd/dev/cicd-github.yaml @@ -256,11 +256,16 @@ Resources: - Name: useVVHGeometry Value: true - Name: vvhRestApiEndPoint - Value: https://haproxy.vayla.fi:2027/vvhdata/ + Value: 'https://api.vayla.fi/vvhdata/' + - Name: vvhRest_username + Value: 'svc_vvh_digiroad' - Name: viiteRestApiEndPoint Value: http://localhost:9080/api/viite/ - Name: authenticationTestMode Value: true + - Name: vvhRest_password + Value: '/dev/vvhRest.password' + Type: PARAMETER_STORE - Name: bonecp_password Value: "/test/bonecp.password" Type: PARAMETER_STORE diff --git a/aws/cloud-formation/parameter-store/digiroad2-parameter-store.yaml b/aws/cloud-formation/parameter-store/digiroad2-parameter-store.yaml index 4eb0bc58b6..826e1ea5fc 100644 --- a/aws/cloud-formation/parameter-store/digiroad2-parameter-store.yaml +++ b/aws/cloud-formation/parameter-store/digiroad2-parameter-store.yaml @@ -64,3 +64,10 @@ Resources: Name: /prod/googlemapapi.crypto_key Type: String Value: PlaceHolderValue + + VvhRestPassword: + Type: AWS::SSM::Parameter + Properties: + Name: /prod/vvhRest.password + Type: String + Value: PlaceHolderValue diff --git a/aws/cloud-formation/taskdefinition/dev-create-taskdefinition.yaml b/aws/cloud-formation/taskdefinition/dev-create-taskdefinition.yaml index a7196db437..4bc6b86608 100644 --- a/aws/cloud-formation/taskdefinition/dev-create-taskdefinition.yaml +++ b/aws/cloud-formation/taskdefinition/dev-create-taskdefinition.yaml @@ -107,9 +107,11 @@ Resources: - Name: useVVHGeometry Value: 'true' - Name: vvhServiceHost - Value: 'haproxy.vayla.fi' + Value: 'api.vayla.fi' - Name: vvhRestApiEndPoint - Value: 'https://haproxy.vayla.fi:2027/vvhdata/' + Value: 'https://api.vayla.fi/vvhdata/' + - Name: vvhRest.username + Value: 'svc_vvh_digiroad' - Name: vvhRoadlink.frozen Value: 'false' - Name: viiteRestApiEndPoint @@ -177,6 +179,8 @@ Resources: ValueFrom: !Sub 'arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/dev/ses_username' - Name: ses.password ValueFrom: !Sub 'arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/dev/ses_password' + - Name: vvhRest.password + ValueFrom: !Sub 'arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/dev/vvhRest.password' - Name: oag.password ValueFrom: !Sub 'arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/dev/authentication.oag.basic.password' - Name: googlemapapi.client_id diff --git a/aws/cloud-formation/taskdefinition/prod-create-taskdefinition.yaml b/aws/cloud-formation/taskdefinition/prod-create-taskdefinition.yaml index a04e1c8d84..c5c5343ad7 100644 --- a/aws/cloud-formation/taskdefinition/prod-create-taskdefinition.yaml +++ b/aws/cloud-formation/taskdefinition/prod-create-taskdefinition.yaml @@ -106,9 +106,11 @@ Resources: - Name: useVVHGeometry Value: 'true' - Name: vvhServiceHost - Value: 'haproxy.vayla.fi' + Value: 'api.vayla.fi' - Name: vvhRestApiEndPoint - Value: 'https://haproxy.vayla.fi:2027/vvhdata/' + Value: 'https://api.vayla.fi/vvhdata/' + - Name: vvhRest.username + Value: 'svc_vvh_digiroad' - Name: vvhRoadlink.frozen Value: 'false' - Name: viiteRestApiEndPoint @@ -174,6 +176,8 @@ Resources: ValueFrom: !Sub 'arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/prod/apikey/mml_map' - Name: ses.username ValueFrom: !Sub 'arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/prod/ses_username' + - Name: vvhRest.password + ValueFrom: !Sub 'arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/prod/vvhRest.password' - Name: ses.password ValueFrom: !Sub 'arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/prod/ses_password' - Name: oag.password diff --git a/aws/cloud-formation/taskdefinition/qa-create-taskdefinition.yaml b/aws/cloud-formation/taskdefinition/qa-create-taskdefinition.yaml index 1ad75a39d5..82e697c4e4 100644 --- a/aws/cloud-formation/taskdefinition/qa-create-taskdefinition.yaml +++ b/aws/cloud-formation/taskdefinition/qa-create-taskdefinition.yaml @@ -108,9 +108,11 @@ Resources: - Name: useVVHGeometry Value: 'true' - Name: vvhServiceHost - Value: 'haproxy.vayla.fi' + Value: 'api.vayla.fi' - Name: vvhRestApiEndPoint - Value: 'https://haproxy.vayla.fi:2027/vvhdata/' + Value: 'https://api.vayla.fi/vvhdata/' + - Name: vvhRest.username + Value: 'svc_vvh_digiroad' - Name: vvhRoadlink.frozen Value: 'false' - Name: viiteRestApiEndPoint @@ -178,6 +180,8 @@ Resources: ValueFrom: !Sub 'arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/qa/ses_username' - Name: ses.password ValueFrom: !Sub 'arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/qa/ses_password' + - Name: vvhRest.password + ValueFrom: !Sub 'arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/qa/vvhRest.password' - Name: oag.password ValueFrom: !Sub 'arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/qa/authentication.oag.basic.password' - Name: googlemapapi.client_id From 8961ce0fa176f2c85ad4383b015fe31a2016d452 Mon Sep 17 00:00:00 2001 From: sasuolander Date: Wed, 18 May 2022 16:26:29 +0300 Subject: [PATCH 24/37] DROTH-3258 update env.properties --- conf/env.properties | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/conf/env.properties b/conf/env.properties index d4ca5503f5..4ad39cb92e 100644 --- a/conf/env.properties +++ b/conf/env.properties @@ -12,8 +12,10 @@ featureProvider=fi.liikennevirasto.digiroad2.service.AssetPropertyService useVVHGeometry=true vvhServiceHost=172.17.204.39:6080 -vvhRestApiEndPoint=http://172.17.204.39:6080/arcgis/rest/services/VVH_OTH/ +vvhRestApiEndPoint=https://api.vayla.fi/vvhdata/ vvhRoadlink.frozen=false +vvhRest.username=svc_vvh_digiroad +vvhRest.password=insertpassword viiteRestApiEndPoint=https://api.testivaylapilvi.fi/viite/api/viite/ viite.apikey=insertapikey From e8e77b65413a9b4eddf0835dccf44d531ddf448b Mon Sep 17 00:00:00 2001 From: sasuolander Date: Wed, 18 May 2022 16:43:26 +0300 Subject: [PATCH 25/37] DROTH-3258 update readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 88fa27bea5..9fa1519f6e 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,7 @@ ses.password=sespassword bonecp.jdbcUrl=kantaurl bonecp.username=kantakäyttäjä bonecp.password=kantasalasana +vvhRest.password=insertpassword ``` Windowsissa toimii komento: ``` From f94ca6467757403a2a63aaaa94f050ec8e708531 Mon Sep 17 00:00:00 2001 From: sasuolander Date: Wed, 18 May 2022 16:49:19 +0300 Subject: [PATCH 26/37] DROTH-3258 update readme --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9fa1519f6e..c9ee931941 100644 --- a/README.md +++ b/README.md @@ -85,9 +85,8 @@ API-palvelimen saa käyntiin kehitysmoodiin seuraavalla sbt komennolla: ./sbt '~;container:start; container:reload /' ``` -Salasanat voidaan syöttää ympäristömuuttujina myös. -Parametri voidaan asettaa Intellij SBT Configuration Environment Variable. -Nämä voidaan syöttään myös ympäristömuuttujina. +Ympäristömuuttuja parametri voidaan asettaa Intellij SBT Configuration Environment Variable avulla. +Nämä voidaan syöttään myös ympäristömuuttujina: ``` viiteRestApiEndPoint=url viite.apikey=insertapikey From 5a66df307774d0aaff0485cb5431cbc9e98042c7 Mon Sep 17 00:00:00 2001 From: "antti.ahopelto" Date: Thu, 19 May 2022 08:40:01 +0300 Subject: [PATCH 27/37] DROTH-3214 Fixed JobDefinition --- .../batchSystem/ProdBatchJobDefinition.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/aws/cloud-formation/batchSystem/ProdBatchJobDefinition.json b/aws/cloud-formation/batchSystem/ProdBatchJobDefinition.json index 28f526df52..acb7c378d1 100644 --- a/aws/cloud-formation/batchSystem/ProdBatchJobDefinition.json +++ b/aws/cloud-formation/batchSystem/ProdBatchJobDefinition.json @@ -11,7 +11,7 @@ "FARGATE" ], "containerProperties": { - "image": "475079312496.dkr.ecr.eu-west-1.amazonaws.com/digiroad2:prod", + "image": "920408837790.dkr.ecr.eu-west-1.amazonaws.com/digiroad2:prod", "resourceRequirements" : [ { "type": "MEMORY", @@ -130,19 +130,19 @@ "secrets": [ { "name": "bonecp.password", - "valueFrom": "arn:aws:ssm:eu-west-1:475079312496:parameter/prod/bonecp.password" + "valueFrom": "arn:aws:ssm:eu-west-1:920408837790:parameter/prod/bonecp.password" }, { "name": "oag.password", - "valueFrom": "arn:aws:ssm:eu-west-1:475079312496:parameter/prod/authentication.oag.basic.password" + "valueFrom": "arn:aws:ssm:eu-west-1:920408837790:parameter/prod/authentication.oag.basic.password" }, { "name": "viite.apikey", - "valueFrom": "arn:aws:ssm:eu-west-1:475079312496:parameter/prod/apikey/viite" + "valueFrom": "arn:aws:ssm:eu-west-1:920408837790:parameter/prod/apikey/viite" }, { "name": "vkm.apikey", - "valueFrom": "arn:aws:ssm:eu-west-1:475079312496:parameter/prod/apikey/viitekehysmuunnin" + "valueFrom": "arn:aws:ssm:eu-west-1:920408837790:parameter/prod/apikey/viitekehysmuunnin" } ] } From 88e866df11fa5b897a133bb38496d087dbd4b9a0 Mon Sep 17 00:00:00 2001 From: "antti.ahopelto" Date: Thu, 19 May 2022 15:00:32 +0300 Subject: [PATCH 28/37] DROTH-3214 batch fail topic and event rule moved to own template, no need for multiple resources on same account --- .../batch-failed-sns-parameter.json | 18 +++++++ .../batchSystem/batchFailedSNS.yaml | 53 +++++++++++++++++++ .../batchSystem/batchSystem.yaml | 42 --------------- .../dev-batch-system-parameter.json | 4 -- .../prod-batch-system-parameter.json | 4 -- .../qa-batch-system-parameter.json | 4 -- 6 files changed, 71 insertions(+), 54 deletions(-) create mode 100644 aws/cloud-formation/batchSystem/batch-failed-sns-parameter.json create mode 100644 aws/cloud-formation/batchSystem/batchFailedSNS.yaml diff --git a/aws/cloud-formation/batchSystem/batch-failed-sns-parameter.json b/aws/cloud-formation/batchSystem/batch-failed-sns-parameter.json new file mode 100644 index 0000000000..27fcd628cb --- /dev/null +++ b/aws/cloud-formation/batchSystem/batch-failed-sns-parameter.json @@ -0,0 +1,18 @@ +[ + { + "ParameterKey": "SNSTopicName", + "ParameterValue": "BatchFailTopic" + }, + { + "ParameterKey": "Owner", + "ParameterValue": "vayla" + }, + { + "ParameterKey": "Project", + "ParameterValue": "digiroad2" + }, + { + "ParameterKey": "ApplicationName", + "ParameterValue": "digiroad2" + } +] \ No newline at end of file diff --git a/aws/cloud-formation/batchSystem/batchFailedSNS.yaml b/aws/cloud-formation/batchSystem/batchFailedSNS.yaml new file mode 100644 index 0000000000..9a61c3bbab --- /dev/null +++ b/aws/cloud-formation/batchSystem/batchFailedSNS.yaml @@ -0,0 +1,53 @@ +AWSTemplateFormatVersion: "2010-09-09" + +Parameters: + SNSTopicName: + Type: String + Description: Name of SNS topic + Owner: + Description: "Owner of the of application, that these resources are created for. Used when tagging the resources" + Type: String + Project: + Description: "Project of the of application, that these resources are created for. Used when tagging the resources" + Type: String + ApplicationName: + Description: Name of the application (no whitespace or special characters) + Type: String + +Resources: + BatchSNSTopic: + Type: AWS::SNS::Topic + Properties: + DisplayName: "AWS Batch Job Has Failed" + FifoTopic: FALSE + Subscription: + - Endpoint: "kehitys@digiroad.fi" + Protocol: "email" + TopicName: !Ref SNSTopicName + Tags: + - Key: Name + Value: !Join [ '-', [ !Ref ApplicationName, 'batchFailTopic' ] ] + - Key: Owner + Value: !Ref Owner + - Key: Project + Value: !Ref Project + + BatchEventRule: + Type: AWS::Events::Rule + Properties: + Description: "Send SNS email for failed batch jobs" + EventPattern: { "detail-type": [ "Batch Job State Change" ], "source": [ "aws.batch" ], "detail": { "status": [ "FAILED" ] } } + Name: 'batch-event-notify' + State: "ENABLED" + Targets: + - Arn: !Join [ '', [ "arn:aws:sns:eu-west-1:", !Ref AWS::AccountId, ":", !Ref SNSTopicName ] ] + Id: "batch-fail-topic" + InputTransformer: + InputTemplate: | + "Batch named in queue failed. Reason was with exit code . Log Stream is " + InputPathsMap: + code: "$.detail.attempts[0].container.exitCode" + log: "$.detail.attempts[0].container.logStreamName" + name: "$.detail.jobName" + queue: "$.detail.jobQueue" + reason: "$.detail.statusReason" \ No newline at end of file diff --git a/aws/cloud-formation/batchSystem/batchSystem.yaml b/aws/cloud-formation/batchSystem/batchSystem.yaml index 37d4fbe23e..4636fe01f7 100644 --- a/aws/cloud-formation/batchSystem/batchSystem.yaml +++ b/aws/cloud-formation/batchSystem/batchSystem.yaml @@ -7,9 +7,6 @@ Parameters: VpcIDOfSystem: Type: AWS::EC2::VPC::Id Description: VPC of your system - SNSTopicName: - Type: String - Description: Name of SNS topic EnvironmentName: Type: String Description: Name of used environment @@ -198,45 +195,6 @@ Resources: - Key: Project Value: !Ref Project - BatchSNSTopic: - Type: AWS::SNS::Topic - Properties: - DisplayName: "AWS Batch Job Has Failed" - FifoTopic: FALSE - Subscription: - - Endpoint: "kehitys@digiroad.fi" - Protocol: "email" - TopicName: !Join [ '-', [ !Ref EnvironmentName, !Ref SNSTopicName ] ] - Tags: - - Key: Name - Value: !Join [ '-', [ !Ref EnvironmentName, !Ref ApplicationName, 'batchFailTopic' ] ] - - Key: Environment - Value: !Ref EnvironmentName - - Key: Owner - Value: !Ref Owner - - Key: Project - Value: !Ref Project - - BatchEventRule: - Type: AWS::Events::Rule - Properties: - Description: "Send SNS email for failed batch jobs" - EventPattern: {"detail-type": ["Batch Job State Change"], "source": ["aws.batch"], "detail": {"status": ["FAILED"]}} - Name: !Join [ '-', [ !Ref EnvironmentName, 'batch-event-notify' ]] - State: "ENABLED" - Targets: - - Arn: !Join ['', ["arn:aws:sns:eu-west-1:", !Ref AWS::AccountId, ":", !Ref EnvironmentName, '-', !Ref SNSTopicName]] - Id: "123321" - InputTransformer: - InputTemplate: | - "Batch named in queue failed. Reason was with exit code . Log Stream is " - InputPathsMap: - code: "$.detail.attempts[0].container.exitCode" - log: "$.detail.attempts[0].container.logStreamName" - name: "$.detail.jobName" - queue: "$.detail.jobQueue" - reason: "$.detail.statusReason" - #Batch Events RunAnnualBatch: Type: AWS::Events::Rule diff --git a/aws/cloud-formation/batchSystem/dev-batch-system-parameter.json b/aws/cloud-formation/batchSystem/dev-batch-system-parameter.json index 106019a523..7f31246f7b 100644 --- a/aws/cloud-formation/batchSystem/dev-batch-system-parameter.json +++ b/aws/cloud-formation/batchSystem/dev-batch-system-parameter.json @@ -7,10 +7,6 @@ "ParameterKey": "VpcIDOfSystem", "ParameterValue": "vpc-0f430b7fedef04ba3" }, - { - "ParameterKey": "SNSTopicName", - "ParameterValue": "BatchFailTopic" - }, { "ParameterKey": "EnvironmentName", "ParameterValue": "DEV" diff --git a/aws/cloud-formation/batchSystem/prod-batch-system-parameter.json b/aws/cloud-formation/batchSystem/prod-batch-system-parameter.json index 68d2f401be..55671e42c8 100644 --- a/aws/cloud-formation/batchSystem/prod-batch-system-parameter.json +++ b/aws/cloud-formation/batchSystem/prod-batch-system-parameter.json @@ -7,10 +7,6 @@ "ParameterKey": "VpcIDOfSystem", "ParameterValue": "vpc-015f75cef3e99c5cc" }, - { - "ParameterKey": "SNSTopicName", - "ParameterValue": "BatchFailTopic" - }, { "ParameterKey": "EnvironmentName", "ParameterValue": "Prod" diff --git a/aws/cloud-formation/batchSystem/qa-batch-system-parameter.json b/aws/cloud-formation/batchSystem/qa-batch-system-parameter.json index b52bd8220e..cb5a905970 100644 --- a/aws/cloud-formation/batchSystem/qa-batch-system-parameter.json +++ b/aws/cloud-formation/batchSystem/qa-batch-system-parameter.json @@ -7,10 +7,6 @@ "ParameterKey": "VpcIDOfSystem", "ParameterValue": "vpc-0b4e33ad8202e91e4" }, - { - "ParameterKey": "SNSTopicName", - "ParameterValue": "BatchFailTopic" - }, { "ParameterKey": "EnvironmentName", "ParameterValue": "QA" From b4e08249dfefb4d4cda850099740d7d7a07ea5a5 Mon Sep 17 00:00:00 2001 From: "antti.ahopelto" Date: Mon, 23 May 2022 08:42:43 +0300 Subject: [PATCH 29/37] DROTH-3214 moved sns files to own directory --- .../{batchSystem => sns}/batch-failed-sns-parameter.json | 0 aws/cloud-formation/{batchSystem => sns}/batchFailedSNS.yaml | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename aws/cloud-formation/{batchSystem => sns}/batch-failed-sns-parameter.json (100%) rename aws/cloud-formation/{batchSystem => sns}/batchFailedSNS.yaml (100%) diff --git a/aws/cloud-formation/batchSystem/batch-failed-sns-parameter.json b/aws/cloud-formation/sns/batch-failed-sns-parameter.json similarity index 100% rename from aws/cloud-formation/batchSystem/batch-failed-sns-parameter.json rename to aws/cloud-formation/sns/batch-failed-sns-parameter.json diff --git a/aws/cloud-formation/batchSystem/batchFailedSNS.yaml b/aws/cloud-formation/sns/batchFailedSNS.yaml similarity index 100% rename from aws/cloud-formation/batchSystem/batchFailedSNS.yaml rename to aws/cloud-formation/sns/batchFailedSNS.yaml From fd450217f13fdb19d1878c00077fa1c0b9ff8323 Mon Sep 17 00:00:00 2001 From: "antti.ahopelto" Date: Mon, 23 May 2022 08:49:22 +0300 Subject: [PATCH 30/37] DROTH-3214 renamed files for future --- .../sns/{batch-failed-sns-parameter.json => sns-parameter.json} | 0 .../sns/{batchFailedSNS.yaml => snsNotifications.yaml} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename aws/cloud-formation/sns/{batch-failed-sns-parameter.json => sns-parameter.json} (100%) rename aws/cloud-formation/sns/{batchFailedSNS.yaml => snsNotifications.yaml} (100%) diff --git a/aws/cloud-formation/sns/batch-failed-sns-parameter.json b/aws/cloud-formation/sns/sns-parameter.json similarity index 100% rename from aws/cloud-formation/sns/batch-failed-sns-parameter.json rename to aws/cloud-formation/sns/sns-parameter.json diff --git a/aws/cloud-formation/sns/batchFailedSNS.yaml b/aws/cloud-formation/sns/snsNotifications.yaml similarity index 100% rename from aws/cloud-formation/sns/batchFailedSNS.yaml rename to aws/cloud-formation/sns/snsNotifications.yaml From 3ce9246f00b2f55f91a246e0c808fcd4a4472b00 Mon Sep 17 00:00:00 2001 From: sasuolander Date: Mon, 23 May 2022 10:28:37 +0300 Subject: [PATCH 31/37] DROTH-3258 update parameter.sh --- aws/cloud-formation/parameter-store/prod-update-parameter.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/aws/cloud-formation/parameter-store/prod-update-parameter.sh b/aws/cloud-formation/parameter-store/prod-update-parameter.sh index 6f20122dcb..537d93abdc 100644 --- a/aws/cloud-formation/parameter-store/prod-update-parameter.sh +++ b/aws/cloud-formation/parameter-store/prod-update-parameter.sh @@ -17,3 +17,4 @@ aws ssm put-parameter --region eu-west-1 --profile vaylaapp --overwrite --name " aws ssm put-parameter --region eu-west-1 --profile vaylaapp --overwrite --name "/prod/authentication.oag.basic.password" --type "SecureString" --value "" aws ssm put-parameter --region eu-west-1 --profile vaylaapp --overwrite --name "/prod/googlemapapi.client_id" --type "SecureString" --value "" aws ssm put-parameter --region eu-west-1 --profile vaylaapp --overwrite --name "/prod/googlemapapi.crypto_key" --type "SecureString" --value "" +aws ssm put-parameter --region eu-west-1 --profile vaylaapp --overwrite --name "/prod/vvhRest.password" --type "SecureString" --value "" From fc7f117b19fa18d74426e32d7dd253fd952a7adc Mon Sep 17 00:00:00 2001 From: "antti.ahopelto" Date: Mon, 23 May 2022 11:42:43 +0300 Subject: [PATCH 32/37] DROTH-3214 renamed parameter --- aws/cloud-formation/sns/sns-parameter.json | 2 +- aws/cloud-formation/sns/snsNotifications.yaml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/aws/cloud-formation/sns/sns-parameter.json b/aws/cloud-formation/sns/sns-parameter.json index 27fcd628cb..e109b38b06 100644 --- a/aws/cloud-formation/sns/sns-parameter.json +++ b/aws/cloud-formation/sns/sns-parameter.json @@ -1,6 +1,6 @@ [ { - "ParameterKey": "SNSTopicName", + "ParameterKey": "BatchSNSTopicName", "ParameterValue": "BatchFailTopic" }, { diff --git a/aws/cloud-formation/sns/snsNotifications.yaml b/aws/cloud-formation/sns/snsNotifications.yaml index 9a61c3bbab..1e900766aa 100644 --- a/aws/cloud-formation/sns/snsNotifications.yaml +++ b/aws/cloud-formation/sns/snsNotifications.yaml @@ -1,7 +1,7 @@ AWSTemplateFormatVersion: "2010-09-09" Parameters: - SNSTopicName: + BatchSNSTopicName: Type: String Description: Name of SNS topic Owner: @@ -23,7 +23,7 @@ Resources: Subscription: - Endpoint: "kehitys@digiroad.fi" Protocol: "email" - TopicName: !Ref SNSTopicName + TopicName: !Ref BatchSNSTopicName Tags: - Key: Name Value: !Join [ '-', [ !Ref ApplicationName, 'batchFailTopic' ] ] @@ -40,7 +40,7 @@ Resources: Name: 'batch-event-notify' State: "ENABLED" Targets: - - Arn: !Join [ '', [ "arn:aws:sns:eu-west-1:", !Ref AWS::AccountId, ":", !Ref SNSTopicName ] ] + - Arn: !Join [ '', [ "arn:aws:sns:eu-west-1:", !Ref AWS::AccountId, ":", !Ref BatchSNSTopicName ] ] Id: "batch-fail-topic" InputTransformer: InputTemplate: | From 0658aa77cbc67c85e800ea869483e04d8f3307f4 Mon Sep 17 00:00:00 2001 From: sasuolander Date: Mon, 23 May 2022 12:12:16 +0300 Subject: [PATCH 33/37] DROTH-3258 fix account id --- aws/cloud-formation/batchSystem/ProdBatchJobDefinition.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/cloud-formation/batchSystem/ProdBatchJobDefinition.json b/aws/cloud-formation/batchSystem/ProdBatchJobDefinition.json index 60c3e47b57..fa50fc4776 100644 --- a/aws/cloud-formation/batchSystem/ProdBatchJobDefinition.json +++ b/aws/cloud-formation/batchSystem/ProdBatchJobDefinition.json @@ -138,7 +138,7 @@ }, { "name": "vvhRest.password", - "valueFrom": "arn:aws:ssm:eu-west-1:475079312496:parameter/prod/vvhRest.password" + "valueFrom": "arn:aws:ssm:eu-west-1:920408837790:parameter/prod/vvhRest.password" }, { "name": "oag.password", From 271affce8d96178f8bdd4612d33c87d5b5d4f8f5 Mon Sep 17 00:00:00 2001 From: "antti.ahopelto" Date: Mon, 23 May 2022 12:20:17 +0300 Subject: [PATCH 34/37] DROTH-3214 fixed Input transformer, added instructions for creating SNS-stack --- aws/cloud-formation/fargateService/prod/README.md | 8 ++++++++ aws/cloud-formation/sns/snsNotifications.yaml | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/aws/cloud-formation/fargateService/prod/README.md b/aws/cloud-formation/fargateService/prod/README.md index c0612c683a..435d4f9d95 100644 --- a/aws/cloud-formation/fargateService/prod/README.md +++ b/aws/cloud-formation/fargateService/prod/README.md @@ -89,6 +89,14 @@ aws cloudformation create-stack \ --parameters file://aws/cloud-formation/fargateService/prod/PROD-alb-ecs-parameter.json ``` +### Luo SNS-ilmoitukset +``` +aws cloudformation create-stack \ +--stack-name SNS-notifications \ +--template-body file://aws/cloud-formation/sns/snsNotifications.yaml \ +--parameters file://aws/cloud-formation/sns/sns-parameter.json +``` + ##Eräajoja varten tuotantotilille luotavat resurssit ### Luo S3 Bucket lambdan koodia varten diff --git a/aws/cloud-formation/sns/snsNotifications.yaml b/aws/cloud-formation/sns/snsNotifications.yaml index 1e900766aa..368ae1f803 100644 --- a/aws/cloud-formation/sns/snsNotifications.yaml +++ b/aws/cloud-formation/sns/snsNotifications.yaml @@ -46,8 +46,8 @@ Resources: InputTemplate: | "Batch named in queue failed. Reason was with exit code . Log Stream is " InputPathsMap: - code: "$.detail.attempts[0].container.exitCode" - log: "$.detail.attempts[0].container.logStreamName" + code: "$.detail.container.exitCode" + log: "$.detail.container.logStreamName" name: "$.detail.jobName" queue: "$.detail.jobQueue" reason: "$.detail.statusReason" \ No newline at end of file From a235d20a13a4c63ec296f505c6b9c43306b9d6ed Mon Sep 17 00:00:00 2001 From: "antti.ahopelto" Date: Wed, 25 May 2022 12:09:42 +0300 Subject: [PATCH 35/37] DROTH-3198 Fixed bug with duplicate changes to link attributes --- .../liikennevirasto/digiroad2/service/RoadLinkService.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/digiroad2-oracle/src/main/scala/fi/liikennevirasto/digiroad2/service/RoadLinkService.scala b/digiroad2-oracle/src/main/scala/fi/liikennevirasto/digiroad2/service/RoadLinkService.scala index 0a17d66f52..e3e7567543 100644 --- a/digiroad2-oracle/src/main/scala/fi/liikennevirasto/digiroad2/service/RoadLinkService.scala +++ b/digiroad2-oracle/src/main/scala/fi/liikennevirasto/digiroad2/service/RoadLinkService.scala @@ -600,9 +600,11 @@ class RoadLinkService(val vvhClient: VVHClient, val eventbus: DigiroadEventBus, } yield (f1Result, f2Result, f3Result) val (complementaryLinks, changes, links) = Await.result(fut, Duration.Inf) + val complementaryLinkIds = complementaryLinks.map(complementaryLink => Some(complementaryLink.linkId)) + val (complementaryChanges, roadLinkChanges) = changes.partition(change => complementaryLinkIds.contains(change.oldId) || complementaryLinkIds.contains(change.newId)) withDynTransaction { - (generateProperties(links, changes), changes, generateProperties(complementaryLinks, changes)) + (generateProperties(links, roadLinkChanges), changes, generateProperties(complementaryLinks, complementaryChanges)) } } From 1419d8b6ef9d7d05648781a5f16dece2f17ce696 Mon Sep 17 00:00:00 2001 From: "antti.ahopelto" Date: Mon, 30 May 2022 09:31:35 +0300 Subject: [PATCH 36/37] DROTH-3198 Added try/catch clause for updating road link attributes --- .../digiroad2/util/UpdateIncompleteLinkList.scala | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/digiroad2-oracle/src/main/scala/fi/liikennevirasto/digiroad2/util/UpdateIncompleteLinkList.scala b/digiroad2-oracle/src/main/scala/fi/liikennevirasto/digiroad2/util/UpdateIncompleteLinkList.scala index 12c7978e72..1a92c35312 100644 --- a/digiroad2-oracle/src/main/scala/fi/liikennevirasto/digiroad2/util/UpdateIncompleteLinkList.scala +++ b/digiroad2-oracle/src/main/scala/fi/liikennevirasto/digiroad2/util/UpdateIncompleteLinkList.scala @@ -13,6 +13,8 @@ import org.slf4j.LoggerFactory import slick.driver.JdbcDriver.backend.Database.dynamicSession import slick.jdbc.StaticQuery.interpolation +import java.sql.SQLException + object UpdateIncompleteLinkList { val logger = LoggerFactory.getLogger(getClass) @@ -116,7 +118,11 @@ object UpdateIncompleteLinkList { val changeSet = RoadLinkChangeSet(pair, stillIncompleteLinksInUse.map(toIncompleteLink), changes, roadLinkDataByLinkId) LogUtils.time(logger, "TEST LOG UpdateRoadLinkChanges") { - updateRoadLinkChanges(changeSet) + try { + updateRoadLinkChanges(changeSet) + } catch { + case sqle: SQLException => logger.error("Updating road link attributes failed with message: " + sqle.getMessage) + } } completeLinks ++ autoGeneratedLinks ++ changedLinks ++ stillIncompleteLinks From 345e62dc7eb4e7b15730c245e1233c02a3e7fa99 Mon Sep 17 00:00:00 2001 From: sasuolander Date: Tue, 31 May 2022 07:57:13 +0300 Subject: [PATCH 37/37] pull image with test tag, default was latest --- aws/cloud-formation/cicd/prod/PROD-cicd.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/aws/cloud-formation/cicd/prod/PROD-cicd.yaml b/aws/cloud-formation/cicd/prod/PROD-cicd.yaml index 5b17bd5746..df48f9058a 100644 --- a/aws/cloud-formation/cicd/prod/PROD-cicd.yaml +++ b/aws/cloud-formation/cicd/prod/PROD-cicd.yaml @@ -73,6 +73,7 @@ Resources: OutputArtifacts: - Name: SourceOutput Configuration: + ImageTag: test RepositoryName: !Ref EcrRepositoryName - Name: Accept Actions: