diff --git a/athena-aws-cmdb/athena-aws-cmdb-package.yaml b/athena-aws-cmdb/athena-aws-cmdb-package.yaml new file mode 100644 index 0000000000..b3265cd1eb --- /dev/null +++ b/athena-aws-cmdb/athena-aws-cmdb-package.yaml @@ -0,0 +1,81 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaAwsCmdbConnector + Description: 'This connector enables Amazon Athena to communicate with various AWS Services, making your resource inventories accessible via SQL.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + AthenaCatalogName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: "WARNING: If set to 'true' encryption for spilled data is disabled." + Default: 'false' + Type: String + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] +Resources: + ConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + FunctionName: !Ref AthenaCatalogName + Handler: "com.amazonaws.athena.connectors.aws.cmdb.AwsCmdbCompositeHandler" + CodeUri: "./target/athena-aws-cmdb-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with various AWS Services, making your resource inventories accessible via SQL." + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - autoscaling:Describe* + - elasticloadbalancing:Describe* + - ec2:Describe* + - elasticmapreduce:Describe* + - elasticmapreduce:List* + - rds:Describe* + - rds:ListTagsForResource + - athena:GetQueryExecution + - s3:ListBucket + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket \ No newline at end of file diff --git a/athena-clickhouse/athena-clickhouse-package.yaml b/athena-clickhouse/athena-clickhouse-package.yaml new file mode 100644 index 0000000000..47aa4cc649 --- /dev/null +++ b/athena-clickhouse/athena-clickhouse-package.yaml @@ -0,0 +1,160 @@ + +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaClickHouseConnector + Description: 'This connector enables Amazon Athena to communicate with your ClickHouse instance(s) using JDBC driver.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + DefaultConnectionString: + Description: 'The default connection string is used when catalog is "lambda:${LambdaFunctionName}". Catalog specific Connection Strings can be added later. Format: ${DatabaseType}://${NativeJdbcConnectionString}.' + Type: String + SecretNamePrefix: + Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena Clickhouse Federation secret names can be prefixed with "AthenaClickHouseFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaJdbcFederation*". Parameter value in this case should be "AthenaClickhouseFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + LambdaRoleARN: + Description: "(Optional) A custom role to be used by the Connector lambda" + Type: String + Default: "" + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: 'One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: 'List' + SubnetIds: + Description: 'One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: 'List' + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + NotHasLambdaRole: !Equals [!Ref LambdaRoleARN, ""] +Resources: + JdbcConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default: !Ref DefaultConnectionString + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.clickhouse.ClickHouseMuxCompositeHandler" + CodeUri: "./target/athena-clickhouse-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with ClickHouse using JDBC" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Role: !If [NotHasLambdaRole, !GetAtt FunctionRole.Arn, !Ref LambdaRoleARN] + VpcConfig: + SecurityGroupIds: !Ref SecurityGroupIds + SubnetIds: !Ref SubnetIds + FunctionRole: + Condition: NotHasLambdaRole + Type: AWS::IAM::Role + Properties: + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - "sts:AssumeRole" + FunctionExecutionPolicy: + Condition: NotHasLambdaRole + Type: "AWS::IAM::Policy" + Properties: + Roles: + - !Ref FunctionRole + PolicyName: FunctionExecutionPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Action: + - secretsmanager:DescribeSecret + - secretsmanager:GetSecretValue + - secretsmanager:GetResourcePolicy + - secretsmanager:ListSecretVersionIds + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:*' + - Action: + - secretsmanager:ListSecrets + Effect: Allow + Resource: '*' + - Action: + - logs:CreateLogGroup + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*' + - Action: + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + - Action: + - ec2:CreateNetworkInterface + - ec2:DeleteNetworkInterface + - ec2:DescribeNetworkInterfaces + - ec2:DetachNetworkInterface + Effect: Allow + Resource: '*' + - Action: + - s3:GetObject + - s3:ListBucket + - s3:GetBucketLocation + - s3:GetObjectVersion + - s3:PutObject + - s3:PutObjectAcl + - s3:GetLifecycleConfiguration + - s3:PutLifecycleConfiguration + - s3:DeleteObject + Effect: Allow + Resource: + - Fn::Sub: + - arn:${AWS::Partition}:s3:::${bucketName} + - bucketName: + Ref: SpillBucket + - Fn::Sub: + - arn:${AWS::Partition}:s3:::${bucketName}/* + - bucketName: + Ref: SpillBucket diff --git a/athena-cloudera-hive/athena-cloudera-hive-package.yaml b/athena-cloudera-hive/athena-cloudera-hive-package.yaml new file mode 100644 index 0000000000..fd56ac05f7 --- /dev/null +++ b/athena-cloudera-hive/athena-cloudera-hive-package.yaml @@ -0,0 +1,109 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaClouderaHiveConnector + Description: 'This connector enables Amazon Athena to communicate with your Cloudera Hive instance(s) using JDBC driver.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + DefaultConnectionString: + Description: 'The default connection string is used when catalog is "lambda:${LambdaFunctionName}". Catalog specific Connection Strings can be added later. Format: ${DatabaseType}://${NativeJdbcConnectionString}.' + Type: String + SecretNamePrefix: + Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena JDBC Federation secret names can be prefixed with "AthenaJdbcFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaJdbcFederation*". Parameter value in this case should be "AthenaJdbcFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: 'One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: 'List' + SubnetIds: + Description: 'One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: 'List' + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] +Resources: + JdbcConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default: !Ref DefaultConnectionString + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.cloudera.HiveMuxCompositeHandler" + CodeUri: "./target/athena-cloudera-hive-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with Coludera Hive using JDBC" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + Version: '2012-10-17' + - Statement: + - Action: + - logs:CreateLogGroup + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' + Version: '2012-10-17' + - Statement: + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*' + Version: '2012-10-17' + - Statement: + - Action: + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket + #VPCAccessPolicy allows our connector to run in a VPC so that it can access your data source. + - VPCAccessPolicy: {} + VpcConfig: + SecurityGroupIds: !Ref SecurityGroupIds + SubnetIds: !Ref SubnetIds \ No newline at end of file diff --git a/athena-cloudera-impala/athena-cloudera-impala-package.yaml b/athena-cloudera-impala/athena-cloudera-impala-package.yaml new file mode 100644 index 0000000000..b26d1c11f1 --- /dev/null +++ b/athena-cloudera-impala/athena-cloudera-impala-package.yaml @@ -0,0 +1,115 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaClouderaImpalaConnector + Description: 'This connector enables Amazon Athena to communicate with your Cloudera Impala instance(s) using JDBC driver.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + DefaultConnectionString: + Description: 'The default connection string is used when catalog is "lambda:${LambdaFunctionName}". Catalog specific Connection Strings can be added later. Format: ${DatabaseType}://${NativeJdbcConnectionString}.' + Type: String + SecretNamePrefix: + Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena JDBC Federation secret names can be prefixed with "AthenaJdbcFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaJdbcFederation*". Parameter value in this case should be "AthenaJdbcFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: 'One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: 'List' + SubnetIds: + Description: 'One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: 'List' + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String + LambdaEncryptionKmsKeyARN: + Description: "(Optional) The KMS Key ARN used for encrypting your Lambda environment variables." + Default: "" + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + HasLambdaEncryptionKmsKeyARN: !Not [ !Equals [ !Ref LambdaEncryptionKmsKeyARN, "" ] ] +Resources: + JdbcConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default: !Ref DefaultConnectionString + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.cloudera.ImpalaMuxCompositeHandler" + CodeUri: "./target/athena-cloudera-impala-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with Cloudera Impala using JDBC" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + Version: '2012-10-17' + - Statement: + - Action: + - logs:CreateLogGroup + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' + Version: '2012-10-17' + - Statement: + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*' + Version: '2012-10-17' + - Statement: + - Action: + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket + #VPCAccessPolicy allows our connector to run in a VPC so that it can access your data source. + - VPCAccessPolicy: {} + VpcConfig: + SecurityGroupIds: !Ref SecurityGroupIds + SubnetIds: !Ref SubnetIds + KmsKeyArn: !If [ HasLambdaEncryptionKmsKeyARN, !Ref LambdaEncryptionKmsKeyARN, !Ref "AWS::NoValue" ] \ No newline at end of file diff --git a/athena-cloudwatch-metrics/athena-cloudwatch-metrics-package.yaml b/athena-cloudwatch-metrics/athena-cloudwatch-metrics-package.yaml new file mode 100644 index 0000000000..d1d815063c --- /dev/null +++ b/athena-cloudwatch-metrics/athena-cloudwatch-metrics-package.yaml @@ -0,0 +1,75 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaCloudwatchMetricsConnector + Description: 'This connector enables Amazon Athena to communicate with Cloudwatch Metrics, making your metrics data accessible via SQL.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + AthenaCatalogName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: "WARNING: If set to 'true' encryption for spilled data is disabled." + Default: 'false' + Type: String + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] +Resources: + ConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + FunctionName: !Ref AthenaCatalogName + Handler: "com.amazonaws.athena.connectors.cloudwatch.metrics.MetricsCompositeHandler" + CodeUri: "./target/athena-cloudwatch-metrics-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with Cloudwatch Metrics, making your metrics data accessible via SQL" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - cloudwatch:Describe* + - cloudwatch:Get* + - cloudwatch:List* + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket \ No newline at end of file diff --git a/athena-cloudwatch/athena-cloudwatch-package.yaml b/athena-cloudwatch/athena-cloudwatch-package.yaml new file mode 100644 index 0000000000..41a7e06439 --- /dev/null +++ b/athena-cloudwatch/athena-cloudwatch-package.yaml @@ -0,0 +1,153 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaCloudwatchConnector + Description: 'This connector enables Amazon Athena to communicate with Cloudwatch, making your logs accessible via SQL.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + AthenaCatalogName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + LambdaRole: + Description: "(Optional) A custom role to be used by the Connector lambda" + Type: String + Default: "" + DisableSpillEncryption: + Description: "WARNING: If set to 'true' encryption for spilled data is disabled." + Default: 'false' + Type: String + KMSKeyId: + Description: "(Optional) By default any data that is spilled to S3 is encrypted using AES-GCM and a randomly generated key. Setting a KMS Key ID allows your Lambda function to use KMS for key generation for a stronger source of encryption keys." + Type: String + Default: "" + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String + +Conditions: + HasKMSKeyId: !Not [!Equals [!Ref KMSKeyId, ""]] + NotHasLambdaRole: !Equals [!Ref LambdaRole, ""] + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + CreateKMSPolicy: !And [ !Condition HasKMSKeyId, !Condition NotHasLambdaRole ] + +Resources: + ConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + kms_key_id: !If [HasKMSKeyId, !Ref KMSKeyId, !Ref "AWS::NoValue"] + FunctionName: !Ref AthenaCatalogName + Handler: "com.amazonaws.athena.connectors.cloudwatch.CloudwatchCompositeHandler" + CodeUri: "./target/athena-cloudwatch-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with Cloudwatch, making your log accessible via SQL" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + Role: !If [NotHasLambdaRole, !GetAtt FunctionRole.Arn, !Ref LambdaRole] + + FunctionRole: + Condition: NotHasLambdaRole + Type: AWS::IAM::Role + Properties: + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - "sts:AssumeRole" + + FunctionExecutionPolicy: + Condition: NotHasLambdaRole + Type: "AWS::IAM::Policy" + Properties: + PolicyName: FunctionExecutionPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Action: + - logs:Describe* + - logs:Get* + - logs:List* + - logs:StartQuery + - logs:StopQuery + - logs:TestMetricFilter + - logs:FilterLogEvents + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + - Action: + - s3:GetObject + - s3:ListBucket + - s3:GetBucketLocation + - s3:GetObjectVersion + - s3:PutObject + - s3:PutObjectAcl + - s3:GetLifecycleConfiguration + - s3:PutLifecycleConfiguration + - s3:DeleteObject + Effect: Allow + Resource: + - Fn::Sub: + - arn:${AWS::Partition}:s3:::${bucketName} + - bucketName: + Ref: SpillBucket + - Fn::Sub: + - arn:${AWS::Partition}:s3:::${bucketName}/* + - bucketName: + Ref: SpillBucket + Roles: + - !Ref FunctionRole + + FunctionKMSPolicy: + Condition: CreateKMSPolicy + Type: "AWS::IAM::Policy" + Properties: + PolicyName: FunctionKMSPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Action: + - kms:GenerateRandom + Effect: Allow + Resource: '*' + - Action: + - kms:GenerateDataKey + Effect: Allow + Resource: !Sub "arn:${AWS::Partition}:kms:${AWS::Region}:${AWS::AccountId}:key/${KMSKeyId}" + Roles: + - !Ref FunctionRole \ No newline at end of file diff --git a/athena-datalakegen2/athena-datalakegen2-package.yaml b/athena-datalakegen2/athena-datalakegen2-package.yaml new file mode 100644 index 0000000000..32da145587 --- /dev/null +++ b/athena-datalakegen2/athena-datalakegen2-package.yaml @@ -0,0 +1,115 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaDataLakeGen2Connector + Description: 'This connector enables Amazon Athena to communicate with DataLake Gen2 using JDBC driver.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - datalake-gen2 + - athena-federation + - jdbc + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + DefaultConnectionString: + Description: 'The default connection string is used when catalog is "lambda:${LambdaFunctionName}". Catalog specific Connection Strings can be added later. Format: ${DatabaseType}://${NativeJdbcConnectionString}.' + Type: String + SecretNamePrefix: + Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena JDBC Federation secret names can be prefixed with "AthenaJdbcFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaJdbcFederation*". Parameter value in this case should be "AthenaJdbcFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: '(Optional) One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: CommaDelimitedList + Default: "" + SubnetIds: + Description: '(Optional) One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: CommaDelimitedList + Default: "" + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] + HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] +Resources: + JdbcConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default: !Ref DefaultConnectionString + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.datalakegen2.DataLakeGen2MuxCompositeHandler" + CodeUri: "./target/athena-datalakegen2-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with DataLake Gen2 using JDBC" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + Version: '2012-10-17' + - Statement: + - Action: + - logs:CreateLogGroup + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' + Version: '2012-10-17' + - Statement: + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*' + Version: '2012-10-17' + - Statement: + - Action: + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket + #VPCAccessPolicy allows our connector to run in a VPC so that it can access your data source. + - VPCAccessPolicy: {} + VpcConfig: + SecurityGroupIds: !If [ HasSecurityGroups, !Ref SecurityGroupIds, !Ref "AWS::NoValue" ] + SubnetIds: !If [ HasSubnets, !Ref SubnetIds, !Ref "AWS::NoValue" ] \ No newline at end of file diff --git a/athena-db2-as400/athena-db2-as400-package.yaml b/athena-db2-as400/athena-db2-as400-package.yaml new file mode 100644 index 0000000000..c84dac623e --- /dev/null +++ b/athena-db2-as400/athena-db2-as400-package.yaml @@ -0,0 +1,116 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaDb2AS400Connector + Description: 'This connector enables Amazon Athena to communicate with DB2 on iSeries (AS400) using JDBC driver.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - ibm + - db2as400 + - athena-federation + - jdbc + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + DefaultConnectionString: + Description: 'The default connection string is used when catalog is "lambda:${LambdaFunctionName}". Catalog specific Connection Strings can be added later. Format: ${DatabaseType}://${NativeJdbcConnectionString}.' + Type: String + SecretNamePrefix: + Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena JDBC Federation secret names can be prefixed with "AthenaJdbcFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaJdbcFederation*". Parameter value in this case should be "AthenaJdbcFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: '(Optional) One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: CommaDelimitedList + Default: "" + SubnetIds: + Description: '(Optional) One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: CommaDelimitedList + Default: "" + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] + HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] +Resources: + JdbcConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default: !Ref DefaultConnectionString + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.db2as400.Db2As400MuxCompositeHandler" + CodeUri: "./target/athena-db2-as400-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with DB2 on iSeries (AS400) using JDBC" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + Version: '2012-10-17' + - Statement: + - Action: + - logs:CreateLogGroup + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' + Version: '2012-10-17' + - Statement: + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*' + Version: '2012-10-17' + - Statement: + - Action: + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket + #VPCAccessPolicy allows our connector to run in a VPC so that it can access your data source. + - VPCAccessPolicy: {} + VpcConfig: + SecurityGroupIds: !If [ HasSecurityGroups, !Ref SecurityGroupIds, !Ref "AWS::NoValue" ] + SubnetIds: !If [ HasSubnets, !Ref SubnetIds, !Ref "AWS::NoValue" ] \ No newline at end of file diff --git a/athena-db2/athena-db2-package.yaml b/athena-db2/athena-db2-package.yaml new file mode 100644 index 0000000000..cbaaa93af9 --- /dev/null +++ b/athena-db2/athena-db2-package.yaml @@ -0,0 +1,116 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaDb2Connector + Description: 'This connector enables Amazon Athena to communicate with DB2 using JDBC driver.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - ibm + - db2 + - athena-federation + - jdbc + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + DefaultConnectionString: + Description: 'The default connection string is used when catalog is "lambda:${LambdaFunctionName}". Catalog specific Connection Strings can be added later. Format: ${DatabaseType}://${NativeJdbcConnectionString}.' + Type: String + SecretNamePrefix: + Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena JDBC Federation secret names can be prefixed with "AthenaJdbcFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaJdbcFederation*". Parameter value in this case should be "AthenaJdbcFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: '(Optional) One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: CommaDelimitedList + Default: "" + SubnetIds: + Description: '(Optional) One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: CommaDelimitedList + Default: "" + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] + HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] +Resources: + JdbcConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default: !Ref DefaultConnectionString + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.db2.Db2MuxCompositeHandler" + CodeUri: "./target/athena-db2-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with DB2 using JDBC" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + Version: '2012-10-17' + - Statement: + - Action: + - logs:CreateLogGroup + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' + Version: '2012-10-17' + - Statement: + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*' + Version: '2012-10-17' + - Statement: + - Action: + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket + #VPCAccessPolicy allows our connector to run in a VPC so that it can access your data source. + - VPCAccessPolicy: {} + VpcConfig: + SecurityGroupIds: !If [ HasSecurityGroups, !Ref SecurityGroupIds, !Ref "AWS::NoValue" ] + SubnetIds: !If [ HasSubnets, !Ref SubnetIds, !Ref "AWS::NoValue" ] \ No newline at end of file diff --git a/athena-docdb/athena-docdb-package.yaml b/athena-docdb/athena-docdb-package.yaml new file mode 100644 index 0000000000..efb7fc0b2e --- /dev/null +++ b/athena-docdb/athena-docdb-package.yaml @@ -0,0 +1,105 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaDocumentDBConnector + Description: This connector enables Amazon Athena to communicate with your DocumentDB instance(s), making your DocumentDB data accessible via SQL. + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + AthenaCatalogName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: 'One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: 'List' + SubnetIds: + Description: 'One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: 'List' + SecretNameOrPrefix: + Description: 'The name or prefix of a set of names within Secrets Manager that this function should have access to. (e.g. hbase-*).' + Type: String + DocDBConnectionString: + Description: 'The DocDB connection details to use by default if not catalog specific connection is defined and optionally using SecretsManager (e.g. ${secret_name}).' + Type: String + Default: "e.g. mongodb://:@:/?ssl=true&ssl_ca_certs=rds-combined-ca-bundle.pem&replicaSet=rs0" + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] +Resources: + ConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default_docdb: !Ref DocDBConnectionString + FunctionName: !Ref AthenaCatalogName + Handler: "com.amazonaws.athena.connectors.docdb.DocDBCompositeHandler" + CodeUri: "./target/athena-docdb-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with DocumentDB, making your DocumentDB data accessible via SQL." + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:*:*:secret:${SecretNameOrPrefix}' + Version: '2012-10-17' + - Statement: + - Action: + - glue:GetTableVersions + - glue:GetPartitions + - glue:GetTables + - glue:GetTableVersion + - glue:GetDatabases + - glue:GetTable + - glue:GetPartition + - glue:GetDatabase + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket + #VPCAccessPolicy allows our connector to run in a VPC so that it can access your data source. + - VPCAccessPolicy: {} + VpcConfig: + SecurityGroupIds: !Ref SecurityGroupIds + SubnetIds: !Ref SubnetIds \ No newline at end of file diff --git a/athena-elasticsearch/athena-elasticsearch-package.yaml b/athena-elasticsearch/athena-elasticsearch-package.yaml new file mode 100644 index 0000000000..9c932f7a85 --- /dev/null +++ b/athena-elasticsearch/athena-elasticsearch-package.yaml @@ -0,0 +1,152 @@ +Transform: 'AWS::Serverless-2016-10-31' + +Metadata: + AWS::ServerlessRepo::Application: + Name: AthenaElasticsearchConnector + Description: 'This connector enables Amazon Athena to communicate with your Elasticsearch instance(s).' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: ['athena-federation'] + HomePageUrl: https://github.com/awslabs/aws-athena-query-federation + SemanticVersion: 2022.47.1 + SourceCodeUrl: https://github.com/awslabs/aws-athena-query-federation + +# Parameters are CloudFormation features to pass input +# to your template when you create a stack +Parameters: + AthenaCatalogName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + SecretNamePrefix: + Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena Elasticsearch Federation secret names can be prefixed with "AthenaESFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaESFederation*". Parameter value in this case should be "AthenaESFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' + Type: String + Default: "" + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: "Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)" + Default: 900 + Type: Number + LambdaMemory: + Description: "Lambda memory in MB (min 128 - 3008 max)." + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: "WARNING: If set to 'true' encryption for spilled data is disabled." + Default: "false" + Type: String + AutoDiscoverEndpoint: + Description: "Set to 'true' when using Amazon Elasticsearch Service. Otherwise, set to 'false' (DomainMapping is required when set to false)." + Default: "true" + Type: String + DomainMapping: + Description: "List of one or more domain-names and their respective endpoints (including secret credentials) in the format: domain1=endpoint1,domain2=endpoint2,... (e.g. movies=https://${secret-credentials}:www.mymovies.com)." + Default: "" + Type: String + QueryTimeoutCluster: + Description: "timeout period (in seconds) for Cluster-Health queries used in the generation of parallel scans (default is 10 seconds)." + Default: 10 + Type: Number + QueryTimeoutSearch: + Description: "timeout period (in seconds) for Search queries used in the retrieval of documents from an index (default is 12 minutes)." + Default: 720 + Type: Number + QueryScrollTimeout: + Description: "timeout period (in seconds) for scroll timeout used in the retrieval of documents (default is 60 seconds)." + Default: 60 + Type: Number + IsVPCAccess: + AllowedValues: + - true + - false + Default: false + Description: "If ElasticSearch cluster is in VPC select true, [true, false] (default is false)" + Type: String + SecurityGroupIds: + Description: '**If IsVPCAccess is True**. Provide one or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: CommaDelimitedList + Default: "" + SubnetIds: + Description: '**If IsVPCAccess is True**. Provide one or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: CommaDelimitedList + Default: "" + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String + +Conditions: + IsVPCAccessSelected: !Equals [!Ref IsVPCAccess, true] + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + +Resources: + ConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + auto_discover_endpoint: !Ref AutoDiscoverEndpoint + domain_mapping: !Ref DomainMapping + query_timeout_cluster: !Ref QueryTimeoutCluster + query_timeout_search: !Ref QueryTimeoutSearch + query_scroll_timeout: !Ref QueryScrollTimeout + FunctionName: !Sub "${AthenaCatalogName}" + Handler: "com.amazonaws.athena.connectors.elasticsearch.ElasticsearchCompositeHandler" + CodeUri: "./target/athena-elasticsearch-2022.47.1.jar" + Description: "The Elasticsearch Lambda Connector provides Athena users the ability to query data stored on Elasticsearch clusters." + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + Version: '2012-10-17' + - Statement: + - Action: + - es:List* + - es:Describe* + - es:ESHttp* + - glue:GetTableVersions + - glue:GetPartitions + - glue:GetTables + - glue:GetTableVersion + - glue:GetDatabases + - glue:GetTable + - glue:GetPartition + - glue:GetDatabase + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket + VpcConfig: + SecurityGroupIds: + !If + - IsVPCAccessSelected + - + !Ref SecurityGroupIds + - !Ref "AWS::NoValue" + SubnetIds: + !If + - IsVPCAccessSelected + - + !Ref SubnetIds + - !Ref "AWS::NoValue" \ No newline at end of file diff --git a/athena-hbase/athena-hbase-package.yaml b/athena-hbase/athena-hbase-package.yaml new file mode 100644 index 0000000000..447f96048f --- /dev/null +++ b/athena-hbase/athena-hbase-package.yaml @@ -0,0 +1,129 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaHBaseConnector + Description: 'This connector enables Amazon Athena to communicate with your HBase instance(s), making your HBase data accessible via SQL.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + AthenaCatalogName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: "WARNING: If set to 'true' encryption for spilled data is disabled." + Default: 'false' + Type: String + SecurityGroupIds: + Description: 'One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: 'List' + SubnetIds: + Description: 'One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: 'List' + SecretNameOrPrefix: + Description: 'The name or prefix of a set of names within Secrets Manager that this function should have access to. (e.g. hbase-*).' + Type: String + HBaseConnectionString: + Description: 'The HBase connection details to use by default in the format: master_hostname:hbase_port:zookeeper_port and optionally using SecretsManager (e.g. ${secret_name}).' + Type: String + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String + KerberosAuthEnabled: + Description: 'Kerberos authentication enabled or not' + Default: "false" + Type: String + KerberosConfigFilesS3Reference: + Description: 'The S3 bucket reference where kerberos auth config files are uploaded. Applicable for Kerberos auth' + Default: "" + Type: String + PrincipalName: + Description: 'Principal name for Kerberos authentication' + Default: "" + Type: String + HbaseRpcProtection: + Description: 'Hbase Rpc Protection value for Kerberos authentication' + Default: "" + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] +Resources: + ConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default_hbase: !Ref HBaseConnectionString + kerberos_auth_enabled: !Ref KerberosAuthEnabled + kerberos_config_files_s3_reference: !Ref KerberosConfigFilesS3Reference + principal_name: !Ref PrincipalName + hbase_rpc_protection: !Ref HbaseRpcProtection + FunctionName: !Ref AthenaCatalogName + Handler: "com.amazonaws.athena.connectors.hbase.HbaseCompositeHandler" + CodeUri: "./target/athena-hbase-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with HBase, making your HBase data accessible via SQL" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:*:*:secret:${SecretNameOrPrefix}' + Version: '2012-10-17' + - Statement: + - Action: + - glue:GetTableVersions + - glue:GetPartitions + - glue:GetTables + - glue:GetTableVersion + - glue:GetDatabases + - glue:GetTable + - glue:GetPartition + - glue:GetDatabase + - athena:GetQueryExecution + - s3:ListBucket + - s3:GetObject + - s3:GetBucketLocation + - s3:GetObjectVersion + - s3:GetLifecycleConfiguration + Effect: Allow + Resource: '*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket + #VPCAccessPolicy allows our connector to run in a VPC so that it can access your data source. + - VPCAccessPolicy: {} + VpcConfig: + SecurityGroupIds: !Ref SecurityGroupIds + SubnetIds: !Ref SubnetIds \ No newline at end of file diff --git a/athena-hortonworks-hive/athena-hortonworks-hive-package.yaml b/athena-hortonworks-hive/athena-hortonworks-hive-package.yaml new file mode 100644 index 0000000000..5de7161e5b --- /dev/null +++ b/athena-hortonworks-hive/athena-hortonworks-hive-package.yaml @@ -0,0 +1,113 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaHortonworksHiveConnector + Description: 'This connector enables Amazon Athena to communicate with your Hortonworks Hive instance(s) using JDBC driver.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + DefaultConnectionString: + Description: 'The default connection string is used when catalog is "lambda:${LambdaFunctionName}". Catalog specific Connection Strings can be added later. Format: ${DatabaseType}://${NativeJdbcConnectionString}.' + Type: String + SecretNamePrefix: + Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena JDBC Federation secret names can be prefixed with "AthenaJdbcFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaJdbcFederation*". Parameter value in this case should be "AthenaJdbcFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: '(Optional) One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: CommaDelimitedList + Default: "" + SubnetIds: + Description: '(Optional) One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: CommaDelimitedList + Default: "" + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] + HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] +Resources: + JdbcConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default: !Ref DefaultConnectionString + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.hortonworks.HiveMuxCompositeHandler" + CodeUri: "./target/athena-hortonworks-hive-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with Hortonworks Hive using JDBC" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + Version: '2012-10-17' + - Statement: + - Action: + - logs:CreateLogGroup + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' + Version: '2012-10-17' + - Statement: + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*' + Version: '2012-10-17' + - Statement: + - Action: + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket + #VPCAccessPolicy allows our connector to run in a VPC so that it can access your data source. + - VPCAccessPolicy: {} + VpcConfig: + SecurityGroupIds: !If [ HasSecurityGroups, !Ref SecurityGroupIds, !Ref "AWS::NoValue" ] + SubnetIds: !If [ HasSubnets, !Ref SubnetIds, !Ref "AWS::NoValue" ] \ No newline at end of file diff --git a/athena-kafka/athena-kafka-package.yaml b/athena-kafka/athena-kafka-package.yaml new file mode 100644 index 0000000000..db99c7f3a5 --- /dev/null +++ b/athena-kafka/athena-kafka-package.yaml @@ -0,0 +1,177 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaKafkaConnector + Description: 'This connector enables Amazon Athena to communicate with Kafka clusters and fetch data.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - kafka + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + AuthType: + Description: 'Authentication details' + Type: String + Default: NO_AUTH + AllowedValues: + - SASL_SSL_SCRAM_SHA512 + - SASL_PLAINTEXT_SCRAM_SHA512 + - SASL_SSL_PLAIN + - SASL_PLAINTEXT_PLAIN + - SSL + - NO_AUTH + KafkaEndpoint: + Description: 'Kafka cluster endpoint' + Type: String + SchemaRegistryUrl: + Description: 'Schema Registry URL. Applicable for Avro/Protobuf Formatted Topics. (syntax. http://:)' + Type: String + Default: "" + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + SecretNamePrefix: + Description: "The secret name within AWS Secrets Manager that contains your aws key and secret Credentials" + Default: "" + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: 'One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: CommaDelimitedList + Default: "" + SubnetIds: + Description: 'One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: CommaDelimitedList + Default: "" + CertificatesS3Reference: + Description: 'The S3 bucket reference where keystore and truststore certificates are uploaded. Applicable for SSL auth' + Default: "" + Type: String + LambdaRoleARN: + Description: "(Optional) A custom role to be used by the Connector lambda" + Default: "" + Type: String + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String + +Conditions: + NotHasLambdaRole: !Equals [!Ref LambdaRoleARN, ""] + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] + HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] + +Resources: + AthenaKafkaConnector: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + secrets_manager_secret: !Ref SecretNamePrefix + certificates_s3_reference: !Ref CertificatesS3Reference + kafka_endpoint: !Ref KafkaEndpoint + schema_registry_url: !Ref SchemaRegistryUrl + auth_type: !Ref AuthType + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.kafka.KafkaCompositeHandler" + CodeUri: "./target/athena-kafka-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with Kafka clusters" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + Role: !If [NotHasLambdaRole, !GetAtt FunctionRole.Arn, !Ref LambdaRoleARN] + VpcConfig: + SecurityGroupIds: !If [ HasSecurityGroups, !Ref SecurityGroupIds, !Ref "AWS::NoValue" ] + SubnetIds: !If [ HasSubnets, !Ref SubnetIds, !Ref "AWS::NoValue"] + + FunctionRole: + Condition: NotHasLambdaRole + Type: AWS::IAM::Role + Properties: + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - "sts:AssumeRole" + + FunctionExecutionPolicy: + Condition: NotHasLambdaRole + Type: "AWS::IAM::Policy" + Properties: + PolicyName: FunctionExecutionPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + - Action: + - logs:CreateLogGroup + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*' + - Action: + - athena:GetQueryExecution + - s3:ListBucket + - s3:GetObject + - s3:GetBucketLocation + - s3:GetObjectVersion + - s3:PutObject + - s3:PutObjectAcl + - s3:GetLifecycleConfiguration + - s3:PutLifecycleConfiguration + - s3:DeleteObject + - kms:Decrypt + - glue:* + Effect: Allow + Resource: '*' + - Action: + - ec2:CreateNetworkInterface + - ec2:DeleteNetworkInterface + - ec2:DescribeNetworkInterfaces + - ec2:DetachNetworkInterface + - autoscaling:CompleteLifecycleAction + Effect: Allow + Resource: '*' + Roles: + - !Ref FunctionRole \ No newline at end of file diff --git a/athena-msk/athena-msk-package.yaml b/athena-msk/athena-msk-package.yaml new file mode 100644 index 0000000000..4adc667169 --- /dev/null +++ b/athena-msk/athena-msk-package.yaml @@ -0,0 +1,173 @@ + +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaMSKConnector + Description: 'This connector enables Amazon Athena to communicate with MSK clusters and fetch data.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - msk + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + AuthType: + Description: 'Authentication details' + Type: String + Default: NO_AUTH + AllowedValues: + - SASL_SSL_AWS_MSK_IAM + - SASL_SSL_SCRAM_SHA512 + - SASL_SSL_PLAIN + - SASL_PLAINTEXT_PLAIN + - SSL + - NO_AUTH + KafkaEndpoint: + Description: 'MSK cluster endpoint' + Type: String + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + SecretNamePrefix: + Description: "The secret name within AWS Secrets Manager that contains your aws key and secret Credentials(Not Required for IAM AUTH)" + Default: "" + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: 'One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: CommaDelimitedList + Default: "" + SubnetIds: + Description: 'One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: CommaDelimitedList + Default: "" + CertificatesS3Reference: + Description: 'The S3 bucket reference where keystore and truststore certificates are uploaded. Applicable for SSL auth' + Default: "" + Type: String + LambdaRoleARN: + Description: "(Must for auth type IAM) A custom role to be used by the Connector lambda" + Default: "" + Type: String + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String + +Conditions: + NotHasLambdaRole: !Equals [!Ref LambdaRoleARN, ""] + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] + HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] + +Resources: + AthenaMSKConnector: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + secrets_manager_secret: !Ref SecretNamePrefix + certificates_s3_reference: !Ref CertificatesS3Reference + kafka_endpoint: !Ref KafkaEndpoint + auth_type: !Ref AuthType + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.msk.AmazonMskCompositeHandler" + CodeUri: "./target/athena-msk-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with MSK clusters" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + Role: !If [NotHasLambdaRole, !GetAtt FunctionRole.Arn, !Ref LambdaRoleARN] + VpcConfig: + SecurityGroupIds: !If [ HasSecurityGroups, !Ref SecurityGroupIds, !Ref "AWS::NoValue" ] + SubnetIds: !If [ HasSubnets, !Ref SubnetIds, !Ref "AWS::NoValue"] + + FunctionRole: + Condition: NotHasLambdaRole + Type: AWS::IAM::Role + Properties: + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - "sts:AssumeRole" + + FunctionExecutionPolicy: + Condition: NotHasLambdaRole + Type: "AWS::IAM::Policy" + Properties: + PolicyName: FunctionExecutionPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + - Action: + - logs:CreateLogGroup + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*' + - Action: + - athena:GetQueryExecution + - s3:ListBucket + - s3:GetObject + - s3:GetBucketLocation + - s3:GetObjectVersion + - s3:PutObject + - s3:PutObjectAcl + - s3:GetLifecycleConfiguration + - s3:PutLifecycleConfiguration + - s3:DeleteObject + - kms:Decrypt + - glue:* + Effect: Allow + Resource: '*' + - Action: + - ec2:CreateNetworkInterface + - ec2:DeleteNetworkInterface + - ec2:DescribeNetworkInterfaces + - ec2:DetachNetworkInterface + - autoscaling:CompleteLifecycleAction + Effect: Allow + Resource: '*' + Roles: + - !Ref FunctionRole diff --git a/athena-mysql/athena-mysql-package.yaml b/athena-mysql/athena-mysql-package.yaml new file mode 100644 index 0000000000..3091da2551 --- /dev/null +++ b/athena-mysql/athena-mysql-package.yaml @@ -0,0 +1,153 @@ + +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaMySQLConnector + Description: 'This connector enables Amazon Athena to communicate with your MySQL instance(s) using JDBC driver.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + DefaultConnectionString: + Description: 'The default connection string is used when catalog is "lambda:${LambdaFunctionName}". Catalog specific Connection Strings can be added later. Format: ${DatabaseType}://${NativeJdbcConnectionString}.' + Type: String + SecretNamePrefix: + Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena MySQL Federation secret names can be prefixed with "AthenaMySQLFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaJdbcFederation*". Parameter value in this case should be "AthenaMySQLFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + LambdaRoleARN: + Description: "(Optional) A custom role to be used by the Connector lambda" + Type: String + Default: "" + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: 'One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: 'List' + SubnetIds: + Description: 'One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: 'List' + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + NotHasLambdaRole: !Equals [!Ref LambdaRoleARN, ""] +Resources: + JdbcConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default: !Ref DefaultConnectionString + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.mysql.MySqlMuxCompositeHandler" + CodeUri: "./target/athena-mysql-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with MySQL using JDBC" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Role: !If [NotHasLambdaRole, !GetAtt FunctionRole.Arn, !Ref LambdaRoleARN] + VpcConfig: + SecurityGroupIds: !Ref SecurityGroupIds + SubnetIds: !Ref SubnetIds + FunctionRole: + Condition: NotHasLambdaRole + Type: AWS::IAM::Role + Properties: + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - "sts:AssumeRole" + FunctionExecutionPolicy: + Condition: NotHasLambdaRole + Type: "AWS::IAM::Policy" + Properties: + Roles: + - !Ref FunctionRole + PolicyName: FunctionExecutionPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + - Action: + - logs:CreateLogGroup + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*' + - Action: + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + - Action: + - ec2:CreateNetworkInterface + - ec2:DeleteNetworkInterface + - ec2:DescribeNetworkInterfaces + - ec2:DetachNetworkInterface + Effect: Allow + Resource: '*' + - Action: + - s3:GetObject + - s3:ListBucket + - s3:GetBucketLocation + - s3:GetObjectVersion + - s3:PutObject + - s3:PutObjectAcl + - s3:GetLifecycleConfiguration + - s3:PutLifecycleConfiguration + - s3:DeleteObject + Effect: Allow + Resource: + - Fn::Sub: + - arn:${AWS::Partition}:s3:::${bucketName} + - bucketName: + Ref: SpillBucket + - Fn::Sub: + - arn:${AWS::Partition}:s3:::${bucketName}/* + - bucketName: + Ref: SpillBucket diff --git a/athena-neptune/athena-neptune-package.yaml b/athena-neptune/athena-neptune-package.yaml new file mode 100644 index 0000000000..c13ed20f89 --- /dev/null +++ b/athena-neptune/athena-neptune-package.yaml @@ -0,0 +1,134 @@ +Transform: 'AWS::Serverless-2016-10-31' + +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaNeptuneConnector + Description: This connector enables Amazon Athena to communicate with your Neptune instance, making your Neptune graph data accessible via SQL. + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: ['athena-federation','athena-neptune','neptune'] + HomePageUrl: https://github.com/awslabs/aws-athena-query-federation + SemanticVersion: 2022.47.1 + SourceCodeUrl: https://github.com/awslabs/aws-athena-query-federation + +Parameters: + NeptuneClusterEndpoint: + Description: 'The Neptune cluster endpoint' + Type: String + NeptunePort: + Description: 'The Neptune port' + Type: String + Default: '8182' + NeptuneClusterResourceID: + Description: 'To find the Neptune cluster resource ID in the Amazon Neptune AWS Management Console, choose the DB cluster that you want. The Resource ID is shown in the Configuration section.' + Type: String + NeptuneGraphType: + Description: 'Type of graph created in Neptune, defaults to PROPERTYGRAPH. Allowed values: PROPERTYGRAPH, RDF' + Type: String + Default: 'PROPERTYGRAPH' + AllowedValues: ["PROPERTYGRAPH", "RDF"] + GlueDatabaseName: + Description: 'Name of the Neptune cluster specific Glue Database that contains schemas of graph vertices' + Type: String + AthenaCatalogName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + Default: 'athena-catalog' + AllowedPattern: ^[a-z0-9-_]{1,64}$ + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-neptune-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: 'One or more SecurityGroup IDs corresponding to the SecurityGroup(s) that should be applied to the Lambda function to allow connectivity to Neptune cluster. (e.g. sg1,sg2,sg3)' + Type: 'List' + SubnetIds: + Description: 'One or more Subnet IDs corresponding to the Subnet(s) that the Lambda function can use to access the Neptune cluster. (e.g. subnet1,subnet2)' + Type: 'List' + IAMEnabled: + Description: 'If set to ''true'' the connector uses Signature Version 4 Signing' + Default: false + Type: String + EnableCaseInsensitiveMatch: + Description: 'If set to ''false'' the connector does a case sensitive match for keys' + Default: true + Type: String + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String + +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + +Resources: + ConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + glue_database_name: !Ref GlueDatabaseName + neptune_endpoint: !Ref NeptuneClusterEndpoint + neptune_port: !Ref NeptunePort + neptune_cluster_res_id: !Ref NeptuneClusterResourceID + iam_enabled: !Ref IAMEnabled + neptune_graphtype: !Ref NeptuneGraphType + SERVICE_REGION: !Ref AWS::Region + enable_caseinsensitivematch: !Ref EnableCaseInsensitiveMatch + FunctionName: !Ref AthenaCatalogName + Handler: "com.amazonaws.athena.connectors.neptune.NeptuneCompositeHandler" + CodeUri: "./target/athena-neptune-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with Neptune, making your Neptune graph data accessible via SQL." + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - glue:GetTableVersions + - glue:GetPartitions + - glue:GetTables + - glue:GetTableVersion + - glue:GetDatabases + - glue:GetTable + - glue:GetPartition + - glue:GetDatabase + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + - Action: + - neptune-db:* + Effect: Allow + #Dynamically construct Neptune Cluster Resource ARN to limit permissions to the specific cluster provided + Resource: !Sub 'arn:${AWS::Partition}:neptune-db:${AWS::Region}:${AWS::AccountId}:${NeptuneClusterResourceID}/*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket + #VPCAccessPolicy allows our connector to run in a VPC so that it can access your data source. + - VPCAccessPolicy: {} + VpcConfig: + SecurityGroupIds: !Ref SecurityGroupIds + SubnetIds: !Ref SubnetIds \ No newline at end of file diff --git a/athena-oracle/athena-oracle-package.yaml b/athena-oracle/athena-oracle-package.yaml new file mode 100644 index 0000000000..76d3b4dca1 --- /dev/null +++ b/athena-oracle/athena-oracle-package.yaml @@ -0,0 +1,158 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaOracleConnector + Description: 'This connector enables Amazon Athena to communicate with your ORACLE instance(s) using JDBC driver.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + DefaultConnectionString: + Description: 'The default connection string is used when catalog is "lambda:${LambdaFunctionName}". Catalog specific Connection Strings can be added later. Format: ${DatabaseType}://${NativeJdbcConnectionString}.' + Type: String + SecretNamePrefix: + Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena JDBC Federation secret names can be prefixed with "AthenaJdbcFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaJdbcFederation*". Parameter value in this case should be "AthenaJdbcFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' + Type: String + IsFIPSEnabled: + AllowedValues: + - true + - false + Default: false + Description: "If oracle rds is FIPS compliance select true, [true, false] (default is false)" + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + LambdaRoleARN: + Description: "(Optional) A custom role to be used by the Connector lambda" + Type: String + Default: "" + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: '(Optional) One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: CommaDelimitedList + Default: "" + SubnetIds: + Description: '(Optional) One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: CommaDelimitedList + Default: "" + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + NotHasLambdaRole: !Equals [!Ref LambdaRoleARN, ""] + HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] + HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] +Resources: + JdbcConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default: !Ref DefaultConnectionString + is_FIPS_Enabled: !Ref IsFIPSEnabled + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.oracle.OracleMuxCompositeHandler" + CodeUri: "./target/athena-oracle-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with ORACLE using JDBC" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Role: !If [NotHasLambdaRole, !GetAtt FunctionRole.Arn, !Ref LambdaRoleARN] + VpcConfig: + SecurityGroupIds: !If [ HasSecurityGroups, !Ref SecurityGroupIds, !Ref "AWS::NoValue" ] + SubnetIds: !If [ HasSubnets, !Ref SubnetIds, !Ref "AWS::NoValue" ] + + FunctionRole: + Condition: NotHasLambdaRole + Type: AWS::IAM::Role + Properties: + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - "sts:AssumeRole" + FunctionExecutionPolicy: + Condition: NotHasLambdaRole + Type: "AWS::IAM::Policy" + Properties: + Roles: + - !Ref FunctionRole + PolicyName: FunctionExecutionPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + - Action: + - logs:CreateLogGroup + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*' + - Action: + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + - Action: + - s3:GetObject + - s3:ListBucket + - s3:GetBucketLocation + - s3:GetObjectVersion + - s3:PutObject + - s3:PutObjectAcl + - s3:GetLifecycleConfiguration + - s3:PutLifecycleConfiguration + - s3:DeleteObject + Effect: Allow + Resource: + - Fn::Sub: + - arn:${AWS::Partition}:s3:::${bucketName} + - bucketName: + Ref: SpillBucket + - Fn::Sub: + - arn:${AWS::Partition}:s3:::${bucketName}/* + - bucketName: + Ref: SpillBucket \ No newline at end of file diff --git a/athena-postgresql/athena-postgresql-package.yaml b/athena-postgresql/athena-postgresql-package.yaml new file mode 100644 index 0000000000..695f0759ca --- /dev/null +++ b/athena-postgresql/athena-postgresql-package.yaml @@ -0,0 +1,165 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaPostgreSQLConnector + Description: 'This connector enables Amazon Athena to communicate with your PostgreSQL instance(s) using JDBC driver.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + DefaultConnectionString: + Description: 'The default connection string is used when catalog is "lambda:${LambdaFunctionName}". Catalog specific Connection Strings can be added later. Format: ${DatabaseType}://${NativeJdbcConnectionString}.' + Type: String + SecretNamePrefix: + Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena PostgreSQL Federation secret names can be prefixed with "AthenaPostgreSQLFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaJdbcFederation*". Parameter value in this case should be "AthenaPostgreSQLFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + LambdaRoleARN: + Description: "(Optional) A custom role to be used by the Connector lambda" + Type: String + Default: "" + SecurityGroupIds: + Description: 'One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: 'List' + SubnetIds: + Description: 'One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: 'List' + CompositeHandler: + Description: 'Use "PostGreSqlMuxCompositeHandler" to access multiple postgres instances and "PostGreSqlCompositeHandler" to access single instance using DefaultConnectionString' + Type: String + Default: "PostGreSqlMuxCompositeHandler" + AllowedValues : ["PostGreSqlMuxCompositeHandler", "PostGreSqlCompositeHandler"] + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String + DefaultScale: + Description: "(Optional) Default value for scale of type Numeric, representing the decimal digits in the fractional part, to the right of the decimal point." + Default: 0 + Type: Number + +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + NotHasLambdaRole: !Equals [!Ref LambdaRoleARN, ""] +Resources: + JdbcConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default: !Ref DefaultConnectionString + default_scale: !Ref DefaultScale + FunctionName: !Ref LambdaFunctionName + Handler: !Sub "com.amazonaws.athena.connectors.postgresql.${CompositeHandler}" + CodeUri: "./target/athena-postgresql-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with PostgreSQL using JDBC" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Role: !If [NotHasLambdaRole, !GetAtt FunctionRole.Arn, !Ref LambdaRoleARN] + VpcConfig: + SecurityGroupIds: !Ref SecurityGroupIds + SubnetIds: !Ref SubnetIds + + FunctionRole: + Condition: NotHasLambdaRole + Type: AWS::IAM::Role + Properties: + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - "sts:AssumeRole" + + FunctionExecutionPolicy: + Condition: NotHasLambdaRole + Type: "AWS::IAM::Policy" + Properties: + Roles: + - !Ref FunctionRole + PolicyName: FunctionExecutionPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + - Action: + - logs:CreateLogGroup + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*' + - Action: + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + - Action: + - ec2:CreateNetworkInterface + - ec2:DeleteNetworkInterface + - ec2:DescribeNetworkInterfaces + - ec2:DetachNetworkInterface + Effect: Allow + Resource: '*' + - Action: + - s3:GetObject + - s3:ListBucket + - s3:GetBucketLocation + - s3:GetObjectVersion + - s3:PutObject + - s3:PutObjectAcl + - s3:GetLifecycleConfiguration + - s3:PutLifecycleConfiguration + - s3:DeleteObject + Effect: Allow + Resource: + - Fn::Sub: + - arn:${AWS::Partition}:s3:::${bucketName} + - bucketName: + Ref: SpillBucket + - Fn::Sub: + - arn:${AWS::Partition}:s3:::${bucketName}/* + - bucketName: + Ref: SpillBucket \ No newline at end of file diff --git a/athena-redis/athena-redis-package.yaml b/athena-redis/athena-redis-package.yaml new file mode 100644 index 0000000000..85ffd47a79 --- /dev/null +++ b/athena-redis/athena-redis-package.yaml @@ -0,0 +1,120 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaRedisConnector + Description: 'This connector enables Amazon Athena to communicate with your Redis instance(s), making your Redis data accessible via SQL.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + AthenaCatalogName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: "WARNING: If set to 'true' encryption for spilled data is disabled." + Default: 'false' + Type: String + SecurityGroupIds: + Description: 'One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: 'List' + SubnetIds: + Description: 'One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: 'List' + SecretNameOrPrefix: + Description: 'The name or prefix of a set of names within Secrets Manager that this function should have access to. (e.g. redis-*).' + Type: String + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String + QPTConnectionEndpoint: + Description: "(Optional) The hostname:port:password of the Redis server that contains data for this table optionally using SecretsManager (e.g. ${secret_name}). Used for Query Pass Through queries only." + Default: '' + Type: String + QPTConnectionSSL: + Description: "(Optional) When True, creates a Redis connection that uses SSL/TLS. Used for Query Pass Through queries only." + Default: 'false' + Type: String + QPTConnectionCluster: + Description: "(Optional) When True, enables support for clustered Redis instances. Used for Query Pass Through queries only." + Default: 'false' + Type: String + QPTConnectionDBNumber: + Description: "(Optional) Set this number (for example 1, 2, or 3) to read from a non-default Redis database. Used for Query Pass Through queries only." + Default: 0 + Type: Number +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] +Resources: + ConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + qpt_endpoint: !Ref QPTConnectionEndpoint + qpt_ssl: !Ref QPTConnectionSSL + qpt_cluster: !Ref QPTConnectionCluster + qpt_db_number: !Ref QPTConnectionDBNumber + FunctionName: !Ref AthenaCatalogName + Handler: "com.amazonaws.athena.connectors.redis.RedisCompositeHandler" + CodeUri: "./target/athena-redis-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with Redis, making your Redis data accessible via SQL" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:*:*:secret:${SecretNameOrPrefix}' + Version: '2012-10-17' + - Statement: + - Action: + - glue:GetTableVersions + - glue:GetPartitions + - glue:GetTables + - glue:GetTableVersion + - glue:GetDatabases + - glue:GetTable + - glue:GetPartition + - glue:GetDatabase + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket + #VPCAccessPolicy allows our connector to run in a VPC so that it can access your data source. + - VPCAccessPolicy: {} + VpcConfig: + SecurityGroupIds: !Ref SecurityGroupIds + SubnetIds: !Ref SubnetIds \ No newline at end of file diff --git a/athena-redshift/athena-redshift-package.yaml b/athena-redshift/athena-redshift-package.yaml new file mode 100644 index 0000000000..7a65432d4b --- /dev/null +++ b/athena-redshift/athena-redshift-package.yaml @@ -0,0 +1,167 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaRedshiftConnector + Description: 'This connector enables Amazon Athena to communicate with your Redshift instance(s) using JDBC driver.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + DefaultConnectionString: + Description: 'The default connection string is used when catalog is "lambda:${LambdaFunctionName}". Catalog specific Connection Strings can be added later. Format: ${DatabaseType}://${NativeJdbcConnectionString}.' + Type: String + SecretNamePrefix: + Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena Redshift Federation secret names can be prefixed with "AthenaRedshiftFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaRedshiftFederation*". Parameter value in this case should be "AthenaRedshiftFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: 'One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: 'List' + SubnetIds: + Description: 'One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: 'List' + KMSKeyId: + Description: "(Optional) By default any data that is spilled to S3 is encrypted using AES-GCM and a randomly generated key. Setting a KMS Key ID allows your Lambda function to use KMS for key generation for a stronger source of encryption keys." + Type: String + Default: "" + LambdaRole: + Description: "(Optional) A custom IAM role ARN to be used by the Connector lambda" + Type: String + Default: "" + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasKMSKeyId: !Not [!Equals [!Ref KMSKeyId, ""]] + NotHasLambdaRole: !Equals [!Ref LambdaRole, ""] + NotHasLambdaRoleAndHasKMSKeyId: !And + - !Condition NotHasLambdaRole + - !Condition HasKMSKeyId + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] +Resources: + JdbcConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default: !Ref DefaultConnectionString + kms_key_id: !If [HasKMSKeyId, !Ref KMSKeyId, !Ref "AWS::NoValue"] + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.redshift.RedshiftMuxCompositeHandler" + CodeUri: "./target/athena-redshift-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with Redshift using JDBC" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + Role: !If [NotHasLambdaRole, !GetAtt FunctionRole.Arn, !Ref LambdaRole] + VpcConfig: + SecurityGroupIds: !Ref SecurityGroupIds + SubnetIds: !Ref SubnetIds + + FunctionRole: + Condition: NotHasLambdaRole + Type: AWS::IAM::Role + Properties: + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - "sts:AssumeRole" + + + FunctionExecutionPolicy: + Condition: NotHasLambdaRole + Type: "AWS::IAM::Policy" + Properties: + PolicyName: FunctionExecutionPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - secretsmanager:GetSecretValue + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + - Effect: Allow + Action: + - athena:GetQueryExecution + Resource: '*' + - Effect: Allow + Action: + - s3:GetObject + - s3:ListBucket + - s3:GetBucketLocation + - s3:GetObjectVersion + - s3:PutObject + - s3:PutObjectAcl + - s3:GetLifecycleConfiguration + - s3:PutLifecycleConfiguration + - s3:DeleteObject + Resource: + - Fn::Sub: + - arn:${AWS::Partition}:s3:::${bucketName} + - bucketName: + Ref: SpillBucket + - Fn::Sub: + - arn:${AWS::Partition}:s3:::${bucketName}/* + - bucketName: + Ref: SpillBucket + Roles: + - !Ref FunctionRole + + FunctionKMSPolicy: + Condition: NotHasLambdaRoleAndHasKMSKeyId + Type: "AWS::IAM::Policy" + Properties: + PolicyName: FunctionKMSPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - kms:GenerateRandom + Resource: '*' + - Effect: Allow + Action: + - kms:GenerateDataKey + Resource: !Sub "arn:${AWS::Partition}:kms:${AWS::Region}:${AWS::AccountId}:key/${KMSKeyId}" + Roles: + - !Ref FunctionRole \ No newline at end of file diff --git a/athena-saphana/athena-saphana-package.yaml b/athena-saphana/athena-saphana-package.yaml new file mode 100644 index 0000000000..d85a95b3eb --- /dev/null +++ b/athena-saphana/athena-saphana-package.yaml @@ -0,0 +1,113 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaSaphanaConnector + Description: 'This connector enables Amazon Athena to communicate with your Teradata instance(s) using JDBC driver.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + DefaultConnectionString: + Description: 'The default connection string is used when catalog is "lambda:${LambdaFunctionName}". Catalog specific Connection Strings can be added later. Format: ${DatabaseType}://${NativeJdbcConnectionString}.' + Type: String + SecretNamePrefix: + Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena JDBC Federation secret names can be prefixed with "AthenaJdbcFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaJdbcFederation*". Parameter value in this case should be "AthenaJdbcFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: '(Optional) One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: CommaDelimitedList + Default: "" + SubnetIds: + Description: '(Optional) One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: CommaDelimitedList + Default: "" + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] + HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] +Resources: + JdbcConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default: !Ref DefaultConnectionString + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.saphana.SaphanaMuxCompositeHandler" + CodeUri: "./target/athena-saphana.zip" + Description: "Enables Amazon Athena to communicate with Teradata using JDBC" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + Version: '2012-10-17' + - Statement: + - Action: + - logs:CreateLogGroup + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' + Version: '2012-10-17' + - Statement: + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*' + Version: '2012-10-17' + - Statement: + - Action: + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket + #VPCAccessPolicy allows our connector to run in a VPC so that it can access your data source. + - VPCAccessPolicy: {} + VpcConfig: + SecurityGroupIds: !If [ HasSecurityGroups, !Ref SecurityGroupIds, !Ref "AWS::NoValue" ] + SubnetIds: !If [ HasSubnets, !Ref SubnetIds, !Ref "AWS::NoValue" ] \ No newline at end of file diff --git a/athena-sqlserver/athena-sqlserver-package.yaml b/athena-sqlserver/athena-sqlserver-package.yaml new file mode 100644 index 0000000000..a560854c7f --- /dev/null +++ b/athena-sqlserver/athena-sqlserver-package.yaml @@ -0,0 +1,158 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaSqlServerConnector + Description: 'This connector enables Amazon Athena to communicate with SQL Server using JDBC driver.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - sqlserver + - athena-federation + - jdbc + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + DefaultConnectionString: + Description: 'The default connection string is used when catalog is "lambda:${LambdaFunctionName}". Catalog specific Connection Strings can be added later. Format: ${DatabaseType}://${NativeJdbcConnectionString}.' + Type: String + SecretNamePrefix: + Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena JDBC Federation secret names can be prefixed with "AthenaJdbcFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaJdbcFederation*". Parameter value in this case should be "AthenaJdbcFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + LambdaRoleARN: + Description: "(Optional) A custom role to be used by the Connector lambda" + Type: String + Default: "" + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: '(Optional) One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: CommaDelimitedList + Default: "" + SubnetIds: + Description: '(Optional) One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: CommaDelimitedList + Default: "" + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + NotHasLambdaRole: !Equals [!Ref LambdaRoleARN, ""] + HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] + HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] +Resources: + JdbcConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default: !Ref DefaultConnectionString + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.sqlserver.SqlServerMuxCompositeHandler" + CodeUri: "./target/athena-sqlserver-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with SQLSERVER using JDBC" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Role: !If [NotHasLambdaRole, !GetAtt FunctionRole.Arn, !Ref LambdaRoleARN] + VpcConfig: + SecurityGroupIds: !If [ HasSecurityGroups, !Ref SecurityGroupIds, !Ref "AWS::NoValue" ] + SubnetIds: !If [ HasSubnets, !Ref SubnetIds, !Ref "AWS::NoValue" ] + FunctionRole: + Condition: NotHasLambdaRole + Type: AWS::IAM::Role + Properties: + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - "sts:AssumeRole" + FunctionExecutionPolicy: + Condition: NotHasLambdaRole + Type: "AWS::IAM::Policy" + Properties: + Roles: + - !Ref FunctionRole + PolicyName: FunctionExecutionPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + - Action: + - logs:CreateLogGroup + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*' + - Action: + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + - Action: + - ec2:CreateNetworkInterface + - ec2:DeleteNetworkInterface + - ec2:DescribeNetworkInterfaces + - ec2:DetachNetworkInterface + Effect: Allow + Resource: '*' + - Action: + - s3:GetObject + - s3:ListBucket + - s3:GetBucketLocation + - s3:GetObjectVersion + - s3:PutObject + - s3:PutObjectAcl + - s3:GetLifecycleConfiguration + - s3:PutLifecycleConfiguration + - s3:DeleteObject + Effect: Allow + Resource: + - Fn::Sub: + - arn:${AWS::Partition}:s3:::${bucketName} + - bucketName: + Ref: SpillBucket + - Fn::Sub: + - arn:${AWS::Partition}:s3:::${bucketName}/* + - bucketName: + Ref: SpillBucket \ No newline at end of file diff --git a/athena-synapse/athena-synapse-package.yaml b/athena-synapse/athena-synapse-package.yaml new file mode 100644 index 0000000000..ccfed7728b --- /dev/null +++ b/athena-synapse/athena-synapse-package.yaml @@ -0,0 +1,147 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaSynapseConnector + Description: 'This connector enables Amazon Athena to communicate with Synapse using JDBC driver.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - synapse + - athena-federation + - jdbc + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + DefaultConnectionString: + Description: 'The default connection string is used when catalog is "lambda:${LambdaFunctionName}". Catalog specific Connection Strings can be added later. Format: ${DatabaseType}://${NativeJdbcConnectionString}.' + Type: String + SecretNamePrefix: + Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena JDBC Federation secret names can be prefixed with "AthenaJdbcFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaJdbcFederation*". Parameter value in this case should be "AthenaJdbcFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + LambdaRoleARN: + Description: "(Optional) A custom role to be used by the Connector lambda" + Default: "" + Type: String + PermissionsBoundaryARN: + Description: "(Optional) A custom Permission Boundary to be used by the Connector lambda" + Default: "" + Type: String + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: '(Optional) One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: CommaDelimitedList + Default: "" + SubnetIds: + Description: '(Optional) One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: CommaDelimitedList + Default: "" + +Conditions: + NotHasLambdaRole: !Equals [!Ref LambdaRoleARN, ""] + HasPermissionsBoundary: !Not [!Equals [!Ref PermissionsBoundaryARN, ""]] + HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] + HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] + +Resources: + JdbcConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default: !Ref DefaultConnectionString + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.synapse.SynapseMuxCompositeHandler" + CodeUri: "./target/athena-synapse-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with SYNPASE using JDBC" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + Role: !If [NotHasLambdaRole, !GetAtt FunctionRole.Arn, !Ref LambdaRoleARN] + VpcConfig: + SecurityGroupIds: !If [ HasSecurityGroups, !Ref SecurityGroupIds, !Ref "AWS::NoValue" ] + SubnetIds: !If [ HasSubnets, !Ref SubnetIds, !Ref "AWS::NoValue" ] + + FunctionRole: + Condition: NotHasLambdaRole + Type: AWS::IAM::Role + Properties: + PermissionsBoundary: + Fn::If: + - HasPermissionsBoundary + - Ref: PermissionsBoundaryARN + - Ref: AWS::NoValue + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - "sts:AssumeRole" + + FunctionExecutionPolicy: + Condition: NotHasLambdaRole + Type: "AWS::IAM::Policy" + Properties: + PolicyName: FunctionExecutionPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + - Action: + - logs:CreateLogGroup + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*' + - Action: + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + - Action: + - ec2:CreateNetworkInterface + - ec2:DeleteNetworkInterface + - ec2:DescribeNetworkInterfaces + - ec2:DetachNetworkInterface + - autoscaling:CompleteLifecycleAction + Effect: Allow + Resource: '*' + Roles: + - !Ref FunctionRole \ No newline at end of file diff --git a/athena-teradata/athena-teradata-package.yaml b/athena-teradata/athena-teradata-package.yaml new file mode 100644 index 0000000000..70cfa6151d --- /dev/null +++ b/athena-teradata/athena-teradata-package.yaml @@ -0,0 +1,118 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaTeradataConnector + Description: 'This connector enables Amazon Athena to communicate with your Teradata instance(s) using JDBC driver.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + DefaultConnectionString: + Description: 'The default connection string is used when catalog is "lambda:${LambdaFunctionName}". Catalog specific Connection Strings can be added later. Format: ${DatabaseType}://${NativeJdbcConnectionString}.' + Type: String + SecretNamePrefix: + Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena JDBC Federation secret names can be prefixed with "AthenaJdbcFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaJdbcFederation*". Parameter value in this case should be "AthenaJdbcFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: '(Optional) One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: CommaDelimitedList + Default: "" + SubnetIds: + Description: '(Optional) One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: CommaDelimitedList + Default: "" + PartitionCount: + Description: 'Partition Count Limit' + Type: Number + Default: 500 + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] + HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] +Resources: + JdbcConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default: !Ref DefaultConnectionString + partitioncount: !Ref PartitionCount + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.teradata.TeradataMuxCompositeHandler" + CodeUri: "./target/athena-teradata-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with Teradata using JDBC" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + Version: '2012-10-17' + - Statement: + - Action: + - logs:CreateLogGroup + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' + Version: '2012-10-17' + - Statement: + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*' + Version: '2012-10-17' + - Statement: + - Action: + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket + #VPCAccessPolicy allows our connector to run in a VPC so that it can access your data source. + - VPCAccessPolicy: {} + VpcConfig: + SecurityGroupIds: !If [ HasSecurityGroups, !Ref SecurityGroupIds, !Ref "AWS::NoValue" ] + SubnetIds: !If [ HasSubnets, !Ref SubnetIds, !Ref "AWS::NoValue" ] \ No newline at end of file diff --git a/athena-timestream/athena-timestream-package.yaml b/athena-timestream/athena-timestream-package.yaml new file mode 100644 index 0000000000..3ee8788678 --- /dev/null +++ b/athena-timestream/athena-timestream-package.yaml @@ -0,0 +1,83 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaTimestreamConnector + Description: 'This connector enables Amazon Athena to communicate with Amazon Timestream, making your time series data accessible from Athena.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + AthenaCatalogName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: "WARNING: If set to 'true' encryption for spilled data is disabled." + Default: 'false' + Type: String + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] +Resources: + ConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + FunctionName: !Ref AthenaCatalogName + Handler: "com.amazonaws.athena.connectors.timestream.TimestreamCompositeHandler" + CodeUri: "./target/athena-timestream-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with Amazon Timestream, making your time series data accessible from Athena." + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - glue:GetTableVersions + - glue:GetPartitions + - glue:GetTables + - glue:GetTableVersion + - glue:GetDatabases + - glue:GetTable + - glue:GetPartition + - glue:GetDatabase + - athena:GetQueryExecution + - timestream:Describe* + - timestream:List* + - timestream:Select* + Effect: Allow + Resource: '*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket \ No newline at end of file diff --git a/athena-tpcds/athena-tpcds-package.yaml b/athena-tpcds/athena-tpcds-package.yaml new file mode 100644 index 0000000000..d2897ac5b8 --- /dev/null +++ b/athena-tpcds/athena-tpcds-package.yaml @@ -0,0 +1,73 @@ + +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaTPCDSConnector + Description: 'This connector enables Amazon Athena to communicate with a randomly generated TPC-DS data source.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + AthenaCatalogName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: "WARNING: If set to 'true' encryption for spilled data is disabled." + Default: 'false' + Type: String + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] +Resources: + ConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + FunctionName: !Ref AthenaCatalogName + Handler: "com.amazonaws.athena.connectors.tpcds.TPCDSCompositeHandler" + CodeUri: "./target/athena-tpcds-2022.47.1.jar" + Description: "This connector enables Amazon Athena to communicate with a randomly generated TPC-DS data source." + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket diff --git a/athena-udfs/athena-udfs-package.yaml b/athena-udfs/athena-udfs-package.yaml new file mode 100644 index 0000000000..3fdea75381 --- /dev/null +++ b/athena-udfs/athena-udfs-package.yaml @@ -0,0 +1,55 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaUserDefinedFunctionsV2 + Description: 'This connector enables Amazon Athena to leverage common UDFs made available via Lambda.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + LambdaFunctionName: + Description: 'The name you will give to Lambda function which executes your UDFs. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + SecretName: + Description: 'The name or prefix of a set of names within Secrets Manager that this function should have access to. (e.g. database-*).' + Type: String + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] +Resources: + ConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.udfs.AthenaUDFHandler" + CodeUri: "./target/athena-udfs-2022.47.1.jar" + Description: "This connector enables Amazon Athena to leverage common UDFs made available via Lambda." + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:*:*:secret:${SecretName}' + Version: '2012-10-17' \ No newline at end of file