From 905347b539a490a7d7d162b0c07ffb40dcc82cff Mon Sep 17 00:00:00 2001 From: Aimery Methena Date: Mon, 3 Feb 2025 16:36:11 -0500 Subject: [PATCH 1/2] add package templates --- athena-aws-cmdb/athena-aws-cmdb-package.yaml | 81 ++++++++ .../athena-clickhouse-package.yaml | 160 ++++++++++++++++ .../athena-cloudera-hive-package.yaml | 109 +++++++++++ .../athena-cloudera-impala-package.yaml | 115 ++++++++++++ .../athena-cloudwatch-metrics-package.yaml | 75 ++++++++ .../athena-cloudwatch-package.yaml | 153 +++++++++++++++ .../athena-datalakegen2-package.yaml | 115 ++++++++++++ .../athena-db2-as400-package.yaml | 116 ++++++++++++ athena-db2/athena-db2-package.yaml | 116 ++++++++++++ athena-docdb/athena-docdb-package.yaml | 105 +++++++++++ .../athena-elasticesearch-package.yaml | 152 +++++++++++++++ athena-gcs/athena-gcs-package.yaml | 101 ++++++++++ athena-hbase/athena-hbase-package.yaml | 129 +++++++++++++ .../athena-hortonworks-hive-package.yaml | 113 +++++++++++ athena-kafka/athena-kafka-package.yaml | 177 ++++++++++++++++++ athena-msk/athena-msk-package.yaml | 173 +++++++++++++++++ athena-mysql/athena-mysql-package.yaml | 153 +++++++++++++++ athena-neptune/athena-neptune-package.yaml | 134 +++++++++++++ athena-oracle/athena-oracle-package.yaml | 158 ++++++++++++++++ .../athena-postgresql-package.yaml | 165 ++++++++++++++++ athena-redis/athena-redis-package.yaml | 120 ++++++++++++ athena-redshift/athena-redshift-package.yaml | 167 +++++++++++++++++ athena-saphana/athena-saphana-package.yaml | 113 +++++++++++ .../athena-sqlserver-package.yaml | 158 ++++++++++++++++ athena-synapse/athena-synapse-package.yaml | 147 +++++++++++++++ athena-teradata/athena-teradata-package.yaml | 123 ++++++++++++ .../athena-timestream-package.yaml | 83 ++++++++ athena-tpcds/athena-tpcds-package.yaml | 73 ++++++++ athena-udfs/athena-udfs-package.yaml | 55 ++++++ athena-vertica/athena-vertica-package.yaml | 129 +++++++++++++ 30 files changed, 3768 insertions(+) create mode 100644 athena-aws-cmdb/athena-aws-cmdb-package.yaml create mode 100644 athena-clickhouse/athena-clickhouse-package.yaml create mode 100644 athena-cloudera-hive/athena-cloudera-hive-package.yaml create mode 100644 athena-cloudera-impala/athena-cloudera-impala-package.yaml create mode 100644 athena-cloudwatch-metrics/athena-cloudwatch-metrics-package.yaml create mode 100644 athena-cloudwatch/athena-cloudwatch-package.yaml create mode 100644 athena-datalakegen2/athena-datalakegen2-package.yaml create mode 100644 athena-db2-as400/athena-db2-as400-package.yaml create mode 100644 athena-db2/athena-db2-package.yaml create mode 100644 athena-docdb/athena-docdb-package.yaml create mode 100644 athena-elasticsearch/athena-elasticesearch-package.yaml create mode 100644 athena-gcs/athena-gcs-package.yaml create mode 100644 athena-hbase/athena-hbase-package.yaml create mode 100644 athena-hortonworks-hive/athena-hortonworks-hive-package.yaml create mode 100644 athena-kafka/athena-kafka-package.yaml create mode 100644 athena-msk/athena-msk-package.yaml create mode 100644 athena-mysql/athena-mysql-package.yaml create mode 100644 athena-neptune/athena-neptune-package.yaml create mode 100644 athena-oracle/athena-oracle-package.yaml create mode 100644 athena-postgresql/athena-postgresql-package.yaml create mode 100644 athena-redis/athena-redis-package.yaml create mode 100644 athena-redshift/athena-redshift-package.yaml create mode 100644 athena-saphana/athena-saphana-package.yaml create mode 100644 athena-sqlserver/athena-sqlserver-package.yaml create mode 100644 athena-synapse/athena-synapse-package.yaml create mode 100644 athena-teradata/athena-teradata-package.yaml create mode 100644 athena-timestream/athena-timestream-package.yaml create mode 100644 athena-tpcds/athena-tpcds-package.yaml create mode 100644 athena-udfs/athena-udfs-package.yaml create mode 100644 athena-vertica/athena-vertica-package.yaml diff --git a/athena-aws-cmdb/athena-aws-cmdb-package.yaml b/athena-aws-cmdb/athena-aws-cmdb-package.yaml new file mode 100644 index 0000000000..b3265cd1eb --- /dev/null +++ b/athena-aws-cmdb/athena-aws-cmdb-package.yaml @@ -0,0 +1,81 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaAwsCmdbConnector + Description: 'This connector enables Amazon Athena to communicate with various AWS Services, making your resource inventories accessible via SQL.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + AthenaCatalogName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: "WARNING: If set to 'true' encryption for spilled data is disabled." + Default: 'false' + Type: String + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] +Resources: + ConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + FunctionName: !Ref AthenaCatalogName + Handler: "com.amazonaws.athena.connectors.aws.cmdb.AwsCmdbCompositeHandler" + CodeUri: "./target/athena-aws-cmdb-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with various AWS Services, making your resource inventories accessible via SQL." + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - autoscaling:Describe* + - elasticloadbalancing:Describe* + - ec2:Describe* + - elasticmapreduce:Describe* + - elasticmapreduce:List* + - rds:Describe* + - rds:ListTagsForResource + - athena:GetQueryExecution + - s3:ListBucket + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket \ No newline at end of file diff --git a/athena-clickhouse/athena-clickhouse-package.yaml b/athena-clickhouse/athena-clickhouse-package.yaml new file mode 100644 index 0000000000..47aa4cc649 --- /dev/null +++ b/athena-clickhouse/athena-clickhouse-package.yaml @@ -0,0 +1,160 @@ + +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaClickHouseConnector + Description: 'This connector enables Amazon Athena to communicate with your ClickHouse instance(s) using JDBC driver.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + DefaultConnectionString: + Description: 'The default connection string is used when catalog is "lambda:${LambdaFunctionName}". Catalog specific Connection Strings can be added later. Format: ${DatabaseType}://${NativeJdbcConnectionString}.' + Type: String + SecretNamePrefix: + Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena Clickhouse Federation secret names can be prefixed with "AthenaClickHouseFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaJdbcFederation*". Parameter value in this case should be "AthenaClickhouseFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + LambdaRoleARN: + Description: "(Optional) A custom role to be used by the Connector lambda" + Type: String + Default: "" + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: 'One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: 'List' + SubnetIds: + Description: 'One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: 'List' + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + NotHasLambdaRole: !Equals [!Ref LambdaRoleARN, ""] +Resources: + JdbcConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default: !Ref DefaultConnectionString + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.clickhouse.ClickHouseMuxCompositeHandler" + CodeUri: "./target/athena-clickhouse-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with ClickHouse using JDBC" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Role: !If [NotHasLambdaRole, !GetAtt FunctionRole.Arn, !Ref LambdaRoleARN] + VpcConfig: + SecurityGroupIds: !Ref SecurityGroupIds + SubnetIds: !Ref SubnetIds + FunctionRole: + Condition: NotHasLambdaRole + Type: AWS::IAM::Role + Properties: + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - "sts:AssumeRole" + FunctionExecutionPolicy: + Condition: NotHasLambdaRole + Type: "AWS::IAM::Policy" + Properties: + Roles: + - !Ref FunctionRole + PolicyName: FunctionExecutionPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Action: + - secretsmanager:DescribeSecret + - secretsmanager:GetSecretValue + - secretsmanager:GetResourcePolicy + - secretsmanager:ListSecretVersionIds + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:*' + - Action: + - secretsmanager:ListSecrets + Effect: Allow + Resource: '*' + - Action: + - logs:CreateLogGroup + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*' + - Action: + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + - Action: + - ec2:CreateNetworkInterface + - ec2:DeleteNetworkInterface + - ec2:DescribeNetworkInterfaces + - ec2:DetachNetworkInterface + Effect: Allow + Resource: '*' + - Action: + - s3:GetObject + - s3:ListBucket + - s3:GetBucketLocation + - s3:GetObjectVersion + - s3:PutObject + - s3:PutObjectAcl + - s3:GetLifecycleConfiguration + - s3:PutLifecycleConfiguration + - s3:DeleteObject + Effect: Allow + Resource: + - Fn::Sub: + - arn:${AWS::Partition}:s3:::${bucketName} + - bucketName: + Ref: SpillBucket + - Fn::Sub: + - arn:${AWS::Partition}:s3:::${bucketName}/* + - bucketName: + Ref: SpillBucket diff --git a/athena-cloudera-hive/athena-cloudera-hive-package.yaml b/athena-cloudera-hive/athena-cloudera-hive-package.yaml new file mode 100644 index 0000000000..fd56ac05f7 --- /dev/null +++ b/athena-cloudera-hive/athena-cloudera-hive-package.yaml @@ -0,0 +1,109 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaClouderaHiveConnector + Description: 'This connector enables Amazon Athena to communicate with your Cloudera Hive instance(s) using JDBC driver.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + DefaultConnectionString: + Description: 'The default connection string is used when catalog is "lambda:${LambdaFunctionName}". Catalog specific Connection Strings can be added later. Format: ${DatabaseType}://${NativeJdbcConnectionString}.' + Type: String + SecretNamePrefix: + Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena JDBC Federation secret names can be prefixed with "AthenaJdbcFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaJdbcFederation*". Parameter value in this case should be "AthenaJdbcFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: 'One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: 'List' + SubnetIds: + Description: 'One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: 'List' + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] +Resources: + JdbcConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default: !Ref DefaultConnectionString + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.cloudera.HiveMuxCompositeHandler" + CodeUri: "./target/athena-cloudera-hive-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with Coludera Hive using JDBC" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + Version: '2012-10-17' + - Statement: + - Action: + - logs:CreateLogGroup + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' + Version: '2012-10-17' + - Statement: + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*' + Version: '2012-10-17' + - Statement: + - Action: + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket + #VPCAccessPolicy allows our connector to run in a VPC so that it can access your data source. + - VPCAccessPolicy: {} + VpcConfig: + SecurityGroupIds: !Ref SecurityGroupIds + SubnetIds: !Ref SubnetIds \ No newline at end of file diff --git a/athena-cloudera-impala/athena-cloudera-impala-package.yaml b/athena-cloudera-impala/athena-cloudera-impala-package.yaml new file mode 100644 index 0000000000..b26d1c11f1 --- /dev/null +++ b/athena-cloudera-impala/athena-cloudera-impala-package.yaml @@ -0,0 +1,115 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaClouderaImpalaConnector + Description: 'This connector enables Amazon Athena to communicate with your Cloudera Impala instance(s) using JDBC driver.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + DefaultConnectionString: + Description: 'The default connection string is used when catalog is "lambda:${LambdaFunctionName}". Catalog specific Connection Strings can be added later. Format: ${DatabaseType}://${NativeJdbcConnectionString}.' + Type: String + SecretNamePrefix: + Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena JDBC Federation secret names can be prefixed with "AthenaJdbcFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaJdbcFederation*". Parameter value in this case should be "AthenaJdbcFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: 'One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: 'List' + SubnetIds: + Description: 'One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: 'List' + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String + LambdaEncryptionKmsKeyARN: + Description: "(Optional) The KMS Key ARN used for encrypting your Lambda environment variables." + Default: "" + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + HasLambdaEncryptionKmsKeyARN: !Not [ !Equals [ !Ref LambdaEncryptionKmsKeyARN, "" ] ] +Resources: + JdbcConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default: !Ref DefaultConnectionString + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.cloudera.ImpalaMuxCompositeHandler" + CodeUri: "./target/athena-cloudera-impala-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with Cloudera Impala using JDBC" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + Version: '2012-10-17' + - Statement: + - Action: + - logs:CreateLogGroup + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' + Version: '2012-10-17' + - Statement: + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*' + Version: '2012-10-17' + - Statement: + - Action: + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket + #VPCAccessPolicy allows our connector to run in a VPC so that it can access your data source. + - VPCAccessPolicy: {} + VpcConfig: + SecurityGroupIds: !Ref SecurityGroupIds + SubnetIds: !Ref SubnetIds + KmsKeyArn: !If [ HasLambdaEncryptionKmsKeyARN, !Ref LambdaEncryptionKmsKeyARN, !Ref "AWS::NoValue" ] \ No newline at end of file diff --git a/athena-cloudwatch-metrics/athena-cloudwatch-metrics-package.yaml b/athena-cloudwatch-metrics/athena-cloudwatch-metrics-package.yaml new file mode 100644 index 0000000000..d1d815063c --- /dev/null +++ b/athena-cloudwatch-metrics/athena-cloudwatch-metrics-package.yaml @@ -0,0 +1,75 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaCloudwatchMetricsConnector + Description: 'This connector enables Amazon Athena to communicate with Cloudwatch Metrics, making your metrics data accessible via SQL.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + AthenaCatalogName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: "WARNING: If set to 'true' encryption for spilled data is disabled." + Default: 'false' + Type: String + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] +Resources: + ConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + FunctionName: !Ref AthenaCatalogName + Handler: "com.amazonaws.athena.connectors.cloudwatch.metrics.MetricsCompositeHandler" + CodeUri: "./target/athena-cloudwatch-metrics-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with Cloudwatch Metrics, making your metrics data accessible via SQL" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - cloudwatch:Describe* + - cloudwatch:Get* + - cloudwatch:List* + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket \ No newline at end of file diff --git a/athena-cloudwatch/athena-cloudwatch-package.yaml b/athena-cloudwatch/athena-cloudwatch-package.yaml new file mode 100644 index 0000000000..41a7e06439 --- /dev/null +++ b/athena-cloudwatch/athena-cloudwatch-package.yaml @@ -0,0 +1,153 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaCloudwatchConnector + Description: 'This connector enables Amazon Athena to communicate with Cloudwatch, making your logs accessible via SQL.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + AthenaCatalogName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + LambdaRole: + Description: "(Optional) A custom role to be used by the Connector lambda" + Type: String + Default: "" + DisableSpillEncryption: + Description: "WARNING: If set to 'true' encryption for spilled data is disabled." + Default: 'false' + Type: String + KMSKeyId: + Description: "(Optional) By default any data that is spilled to S3 is encrypted using AES-GCM and a randomly generated key. Setting a KMS Key ID allows your Lambda function to use KMS for key generation for a stronger source of encryption keys." + Type: String + Default: "" + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String + +Conditions: + HasKMSKeyId: !Not [!Equals [!Ref KMSKeyId, ""]] + NotHasLambdaRole: !Equals [!Ref LambdaRole, ""] + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + CreateKMSPolicy: !And [ !Condition HasKMSKeyId, !Condition NotHasLambdaRole ] + +Resources: + ConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + kms_key_id: !If [HasKMSKeyId, !Ref KMSKeyId, !Ref "AWS::NoValue"] + FunctionName: !Ref AthenaCatalogName + Handler: "com.amazonaws.athena.connectors.cloudwatch.CloudwatchCompositeHandler" + CodeUri: "./target/athena-cloudwatch-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with Cloudwatch, making your log accessible via SQL" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + Role: !If [NotHasLambdaRole, !GetAtt FunctionRole.Arn, !Ref LambdaRole] + + FunctionRole: + Condition: NotHasLambdaRole + Type: AWS::IAM::Role + Properties: + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - "sts:AssumeRole" + + FunctionExecutionPolicy: + Condition: NotHasLambdaRole + Type: "AWS::IAM::Policy" + Properties: + PolicyName: FunctionExecutionPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Action: + - logs:Describe* + - logs:Get* + - logs:List* + - logs:StartQuery + - logs:StopQuery + - logs:TestMetricFilter + - logs:FilterLogEvents + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + - Action: + - s3:GetObject + - s3:ListBucket + - s3:GetBucketLocation + - s3:GetObjectVersion + - s3:PutObject + - s3:PutObjectAcl + - s3:GetLifecycleConfiguration + - s3:PutLifecycleConfiguration + - s3:DeleteObject + Effect: Allow + Resource: + - Fn::Sub: + - arn:${AWS::Partition}:s3:::${bucketName} + - bucketName: + Ref: SpillBucket + - Fn::Sub: + - arn:${AWS::Partition}:s3:::${bucketName}/* + - bucketName: + Ref: SpillBucket + Roles: + - !Ref FunctionRole + + FunctionKMSPolicy: + Condition: CreateKMSPolicy + Type: "AWS::IAM::Policy" + Properties: + PolicyName: FunctionKMSPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Action: + - kms:GenerateRandom + Effect: Allow + Resource: '*' + - Action: + - kms:GenerateDataKey + Effect: Allow + Resource: !Sub "arn:${AWS::Partition}:kms:${AWS::Region}:${AWS::AccountId}:key/${KMSKeyId}" + Roles: + - !Ref FunctionRole \ No newline at end of file diff --git a/athena-datalakegen2/athena-datalakegen2-package.yaml b/athena-datalakegen2/athena-datalakegen2-package.yaml new file mode 100644 index 0000000000..32da145587 --- /dev/null +++ b/athena-datalakegen2/athena-datalakegen2-package.yaml @@ -0,0 +1,115 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaDataLakeGen2Connector + Description: 'This connector enables Amazon Athena to communicate with DataLake Gen2 using JDBC driver.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - datalake-gen2 + - athena-federation + - jdbc + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + DefaultConnectionString: + Description: 'The default connection string is used when catalog is "lambda:${LambdaFunctionName}". Catalog specific Connection Strings can be added later. Format: ${DatabaseType}://${NativeJdbcConnectionString}.' + Type: String + SecretNamePrefix: + Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena JDBC Federation secret names can be prefixed with "AthenaJdbcFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaJdbcFederation*". Parameter value in this case should be "AthenaJdbcFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: '(Optional) One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: CommaDelimitedList + Default: "" + SubnetIds: + Description: '(Optional) One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: CommaDelimitedList + Default: "" + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] + HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] +Resources: + JdbcConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default: !Ref DefaultConnectionString + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.datalakegen2.DataLakeGen2MuxCompositeHandler" + CodeUri: "./target/athena-datalakegen2-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with DataLake Gen2 using JDBC" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + Version: '2012-10-17' + - Statement: + - Action: + - logs:CreateLogGroup + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' + Version: '2012-10-17' + - Statement: + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*' + Version: '2012-10-17' + - Statement: + - Action: + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket + #VPCAccessPolicy allows our connector to run in a VPC so that it can access your data source. + - VPCAccessPolicy: {} + VpcConfig: + SecurityGroupIds: !If [ HasSecurityGroups, !Ref SecurityGroupIds, !Ref "AWS::NoValue" ] + SubnetIds: !If [ HasSubnets, !Ref SubnetIds, !Ref "AWS::NoValue" ] \ No newline at end of file diff --git a/athena-db2-as400/athena-db2-as400-package.yaml b/athena-db2-as400/athena-db2-as400-package.yaml new file mode 100644 index 0000000000..c84dac623e --- /dev/null +++ b/athena-db2-as400/athena-db2-as400-package.yaml @@ -0,0 +1,116 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaDb2AS400Connector + Description: 'This connector enables Amazon Athena to communicate with DB2 on iSeries (AS400) using JDBC driver.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - ibm + - db2as400 + - athena-federation + - jdbc + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + DefaultConnectionString: + Description: 'The default connection string is used when catalog is "lambda:${LambdaFunctionName}". Catalog specific Connection Strings can be added later. Format: ${DatabaseType}://${NativeJdbcConnectionString}.' + Type: String + SecretNamePrefix: + Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena JDBC Federation secret names can be prefixed with "AthenaJdbcFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaJdbcFederation*". Parameter value in this case should be "AthenaJdbcFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: '(Optional) One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: CommaDelimitedList + Default: "" + SubnetIds: + Description: '(Optional) One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: CommaDelimitedList + Default: "" + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] + HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] +Resources: + JdbcConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default: !Ref DefaultConnectionString + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.db2as400.Db2As400MuxCompositeHandler" + CodeUri: "./target/athena-db2-as400-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with DB2 on iSeries (AS400) using JDBC" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + Version: '2012-10-17' + - Statement: + - Action: + - logs:CreateLogGroup + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' + Version: '2012-10-17' + - Statement: + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*' + Version: '2012-10-17' + - Statement: + - Action: + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket + #VPCAccessPolicy allows our connector to run in a VPC so that it can access your data source. + - VPCAccessPolicy: {} + VpcConfig: + SecurityGroupIds: !If [ HasSecurityGroups, !Ref SecurityGroupIds, !Ref "AWS::NoValue" ] + SubnetIds: !If [ HasSubnets, !Ref SubnetIds, !Ref "AWS::NoValue" ] \ No newline at end of file diff --git a/athena-db2/athena-db2-package.yaml b/athena-db2/athena-db2-package.yaml new file mode 100644 index 0000000000..cbaaa93af9 --- /dev/null +++ b/athena-db2/athena-db2-package.yaml @@ -0,0 +1,116 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaDb2Connector + Description: 'This connector enables Amazon Athena to communicate with DB2 using JDBC driver.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - ibm + - db2 + - athena-federation + - jdbc + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + DefaultConnectionString: + Description: 'The default connection string is used when catalog is "lambda:${LambdaFunctionName}". Catalog specific Connection Strings can be added later. Format: ${DatabaseType}://${NativeJdbcConnectionString}.' + Type: String + SecretNamePrefix: + Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena JDBC Federation secret names can be prefixed with "AthenaJdbcFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaJdbcFederation*". Parameter value in this case should be "AthenaJdbcFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: '(Optional) One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: CommaDelimitedList + Default: "" + SubnetIds: + Description: '(Optional) One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: CommaDelimitedList + Default: "" + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] + HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] +Resources: + JdbcConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default: !Ref DefaultConnectionString + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.db2.Db2MuxCompositeHandler" + CodeUri: "./target/athena-db2-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with DB2 using JDBC" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + Version: '2012-10-17' + - Statement: + - Action: + - logs:CreateLogGroup + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' + Version: '2012-10-17' + - Statement: + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*' + Version: '2012-10-17' + - Statement: + - Action: + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket + #VPCAccessPolicy allows our connector to run in a VPC so that it can access your data source. + - VPCAccessPolicy: {} + VpcConfig: + SecurityGroupIds: !If [ HasSecurityGroups, !Ref SecurityGroupIds, !Ref "AWS::NoValue" ] + SubnetIds: !If [ HasSubnets, !Ref SubnetIds, !Ref "AWS::NoValue" ] \ No newline at end of file diff --git a/athena-docdb/athena-docdb-package.yaml b/athena-docdb/athena-docdb-package.yaml new file mode 100644 index 0000000000..efb7fc0b2e --- /dev/null +++ b/athena-docdb/athena-docdb-package.yaml @@ -0,0 +1,105 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaDocumentDBConnector + Description: This connector enables Amazon Athena to communicate with your DocumentDB instance(s), making your DocumentDB data accessible via SQL. + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + AthenaCatalogName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: 'One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: 'List' + SubnetIds: + Description: 'One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: 'List' + SecretNameOrPrefix: + Description: 'The name or prefix of a set of names within Secrets Manager that this function should have access to. (e.g. hbase-*).' + Type: String + DocDBConnectionString: + Description: 'The DocDB connection details to use by default if not catalog specific connection is defined and optionally using SecretsManager (e.g. ${secret_name}).' + Type: String + Default: "e.g. mongodb://:@:/?ssl=true&ssl_ca_certs=rds-combined-ca-bundle.pem&replicaSet=rs0" + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] +Resources: + ConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default_docdb: !Ref DocDBConnectionString + FunctionName: !Ref AthenaCatalogName + Handler: "com.amazonaws.athena.connectors.docdb.DocDBCompositeHandler" + CodeUri: "./target/athena-docdb-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with DocumentDB, making your DocumentDB data accessible via SQL." + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:*:*:secret:${SecretNameOrPrefix}' + Version: '2012-10-17' + - Statement: + - Action: + - glue:GetTableVersions + - glue:GetPartitions + - glue:GetTables + - glue:GetTableVersion + - glue:GetDatabases + - glue:GetTable + - glue:GetPartition + - glue:GetDatabase + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket + #VPCAccessPolicy allows our connector to run in a VPC so that it can access your data source. + - VPCAccessPolicy: {} + VpcConfig: + SecurityGroupIds: !Ref SecurityGroupIds + SubnetIds: !Ref SubnetIds \ No newline at end of file diff --git a/athena-elasticsearch/athena-elasticesearch-package.yaml b/athena-elasticsearch/athena-elasticesearch-package.yaml new file mode 100644 index 0000000000..9c932f7a85 --- /dev/null +++ b/athena-elasticsearch/athena-elasticesearch-package.yaml @@ -0,0 +1,152 @@ +Transform: 'AWS::Serverless-2016-10-31' + +Metadata: + AWS::ServerlessRepo::Application: + Name: AthenaElasticsearchConnector + Description: 'This connector enables Amazon Athena to communicate with your Elasticsearch instance(s).' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: ['athena-federation'] + HomePageUrl: https://github.com/awslabs/aws-athena-query-federation + SemanticVersion: 2022.47.1 + SourceCodeUrl: https://github.com/awslabs/aws-athena-query-federation + +# Parameters are CloudFormation features to pass input +# to your template when you create a stack +Parameters: + AthenaCatalogName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + SecretNamePrefix: + Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena Elasticsearch Federation secret names can be prefixed with "AthenaESFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaESFederation*". Parameter value in this case should be "AthenaESFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' + Type: String + Default: "" + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: "Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)" + Default: 900 + Type: Number + LambdaMemory: + Description: "Lambda memory in MB (min 128 - 3008 max)." + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: "WARNING: If set to 'true' encryption for spilled data is disabled." + Default: "false" + Type: String + AutoDiscoverEndpoint: + Description: "Set to 'true' when using Amazon Elasticsearch Service. Otherwise, set to 'false' (DomainMapping is required when set to false)." + Default: "true" + Type: String + DomainMapping: + Description: "List of one or more domain-names and their respective endpoints (including secret credentials) in the format: domain1=endpoint1,domain2=endpoint2,... (e.g. movies=https://${secret-credentials}:www.mymovies.com)." + Default: "" + Type: String + QueryTimeoutCluster: + Description: "timeout period (in seconds) for Cluster-Health queries used in the generation of parallel scans (default is 10 seconds)." + Default: 10 + Type: Number + QueryTimeoutSearch: + Description: "timeout period (in seconds) for Search queries used in the retrieval of documents from an index (default is 12 minutes)." + Default: 720 + Type: Number + QueryScrollTimeout: + Description: "timeout period (in seconds) for scroll timeout used in the retrieval of documents (default is 60 seconds)." + Default: 60 + Type: Number + IsVPCAccess: + AllowedValues: + - true + - false + Default: false + Description: "If ElasticSearch cluster is in VPC select true, [true, false] (default is false)" + Type: String + SecurityGroupIds: + Description: '**If IsVPCAccess is True**. Provide one or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: CommaDelimitedList + Default: "" + SubnetIds: + Description: '**If IsVPCAccess is True**. Provide one or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: CommaDelimitedList + Default: "" + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String + +Conditions: + IsVPCAccessSelected: !Equals [!Ref IsVPCAccess, true] + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + +Resources: + ConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + auto_discover_endpoint: !Ref AutoDiscoverEndpoint + domain_mapping: !Ref DomainMapping + query_timeout_cluster: !Ref QueryTimeoutCluster + query_timeout_search: !Ref QueryTimeoutSearch + query_scroll_timeout: !Ref QueryScrollTimeout + FunctionName: !Sub "${AthenaCatalogName}" + Handler: "com.amazonaws.athena.connectors.elasticsearch.ElasticsearchCompositeHandler" + CodeUri: "./target/athena-elasticsearch-2022.47.1.jar" + Description: "The Elasticsearch Lambda Connector provides Athena users the ability to query data stored on Elasticsearch clusters." + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + Version: '2012-10-17' + - Statement: + - Action: + - es:List* + - es:Describe* + - es:ESHttp* + - glue:GetTableVersions + - glue:GetPartitions + - glue:GetTables + - glue:GetTableVersion + - glue:GetDatabases + - glue:GetTable + - glue:GetPartition + - glue:GetDatabase + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket + VpcConfig: + SecurityGroupIds: + !If + - IsVPCAccessSelected + - + !Ref SecurityGroupIds + - !Ref "AWS::NoValue" + SubnetIds: + !If + - IsVPCAccessSelected + - + !Ref SubnetIds + - !Ref "AWS::NoValue" \ No newline at end of file diff --git a/athena-gcs/athena-gcs-package.yaml b/athena-gcs/athena-gcs-package.yaml new file mode 100644 index 0000000000..621998ba81 --- /dev/null +++ b/athena-gcs/athena-gcs-package.yaml @@ -0,0 +1,101 @@ +Transform: 'AWS::Serverless-2016-10-31' + +Metadata: + AWS::ServerlessRepo::Application: + Name: AthenaGCSConnector + Description: 'This connector enables Amazon Athena to communicate with Google Cloud Storage (GCS) and fetch data from Parquet file format' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: ['athena-federation', 'GCS', 'Google-Cloud-Storage', 'parquet', 'csv'] + HomePageUrl: https://github.com/awslabs/aws-athena-query-federation + SemanticVersion: 2022.47.1 + SourceCodeUrl: https://github.com/awslabs/aws-athena-query-federation + +Parameters: + LambdaFunctionName: + Description: 'The name you will give to this catalog is a Lambda name for Athena. Athena will use this name as the function name. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: "Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)" + Default: 900 + Type: Number + LambdaMemory: + Description: "Lambda memory in MB (min 128 - 3008 max)." + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: "WARNING: If set to 'true' encryption for spilled data is disabled." + Default: "false" + Type: String + GCSSecretName: + Description: 'Secret key name in the AWS Secrets Manager.' + Type: String + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String + +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + +Resources: + AthenaGCSConnector: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + secret_manager_gcp_creds_name: !Ref GCSSecretName + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.gcs.GcsCompositeHandler" + CodeUri: "./target/athena-gcs.zip" + Description: "Amazon Athena GCS Connector" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - athena:GetQueryExecution + - glue:GetTableVersions + - glue:GetPartitions + - glue:GetTables + - glue:GetTableVersion + - glue:GetDatabases + - glue:GetTable + - glue:GetPartition + - glue:GetDatabase + Effect: Allow + Resource: '*' + Version: '2012-10-17' + - Statement: + - Action: + - s3:ListBucket + Effect: Allow + Resource: + - !Sub 'arn:${AWS::Partition}:s3:::${SpillBucket}' + Version: '2012-10-17' + - Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:*:*:secret:${GCSSecretName}*' + - S3ReadPolicy: + BucketName: + Ref: SpillBucket + - S3WritePolicy: + BucketName: + Ref: SpillBucket \ No newline at end of file diff --git a/athena-hbase/athena-hbase-package.yaml b/athena-hbase/athena-hbase-package.yaml new file mode 100644 index 0000000000..447f96048f --- /dev/null +++ b/athena-hbase/athena-hbase-package.yaml @@ -0,0 +1,129 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaHBaseConnector + Description: 'This connector enables Amazon Athena to communicate with your HBase instance(s), making your HBase data accessible via SQL.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + AthenaCatalogName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: "WARNING: If set to 'true' encryption for spilled data is disabled." + Default: 'false' + Type: String + SecurityGroupIds: + Description: 'One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: 'List' + SubnetIds: + Description: 'One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: 'List' + SecretNameOrPrefix: + Description: 'The name or prefix of a set of names within Secrets Manager that this function should have access to. (e.g. hbase-*).' + Type: String + HBaseConnectionString: + Description: 'The HBase connection details to use by default in the format: master_hostname:hbase_port:zookeeper_port and optionally using SecretsManager (e.g. ${secret_name}).' + Type: String + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String + KerberosAuthEnabled: + Description: 'Kerberos authentication enabled or not' + Default: "false" + Type: String + KerberosConfigFilesS3Reference: + Description: 'The S3 bucket reference where kerberos auth config files are uploaded. Applicable for Kerberos auth' + Default: "" + Type: String + PrincipalName: + Description: 'Principal name for Kerberos authentication' + Default: "" + Type: String + HbaseRpcProtection: + Description: 'Hbase Rpc Protection value for Kerberos authentication' + Default: "" + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] +Resources: + ConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default_hbase: !Ref HBaseConnectionString + kerberos_auth_enabled: !Ref KerberosAuthEnabled + kerberos_config_files_s3_reference: !Ref KerberosConfigFilesS3Reference + principal_name: !Ref PrincipalName + hbase_rpc_protection: !Ref HbaseRpcProtection + FunctionName: !Ref AthenaCatalogName + Handler: "com.amazonaws.athena.connectors.hbase.HbaseCompositeHandler" + CodeUri: "./target/athena-hbase-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with HBase, making your HBase data accessible via SQL" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:*:*:secret:${SecretNameOrPrefix}' + Version: '2012-10-17' + - Statement: + - Action: + - glue:GetTableVersions + - glue:GetPartitions + - glue:GetTables + - glue:GetTableVersion + - glue:GetDatabases + - glue:GetTable + - glue:GetPartition + - glue:GetDatabase + - athena:GetQueryExecution + - s3:ListBucket + - s3:GetObject + - s3:GetBucketLocation + - s3:GetObjectVersion + - s3:GetLifecycleConfiguration + Effect: Allow + Resource: '*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket + #VPCAccessPolicy allows our connector to run in a VPC so that it can access your data source. + - VPCAccessPolicy: {} + VpcConfig: + SecurityGroupIds: !Ref SecurityGroupIds + SubnetIds: !Ref SubnetIds \ No newline at end of file diff --git a/athena-hortonworks-hive/athena-hortonworks-hive-package.yaml b/athena-hortonworks-hive/athena-hortonworks-hive-package.yaml new file mode 100644 index 0000000000..5de7161e5b --- /dev/null +++ b/athena-hortonworks-hive/athena-hortonworks-hive-package.yaml @@ -0,0 +1,113 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaHortonworksHiveConnector + Description: 'This connector enables Amazon Athena to communicate with your Hortonworks Hive instance(s) using JDBC driver.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + DefaultConnectionString: + Description: 'The default connection string is used when catalog is "lambda:${LambdaFunctionName}". Catalog specific Connection Strings can be added later. Format: ${DatabaseType}://${NativeJdbcConnectionString}.' + Type: String + SecretNamePrefix: + Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena JDBC Federation secret names can be prefixed with "AthenaJdbcFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaJdbcFederation*". Parameter value in this case should be "AthenaJdbcFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: '(Optional) One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: CommaDelimitedList + Default: "" + SubnetIds: + Description: '(Optional) One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: CommaDelimitedList + Default: "" + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] + HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] +Resources: + JdbcConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default: !Ref DefaultConnectionString + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.hortonworks.HiveMuxCompositeHandler" + CodeUri: "./target/athena-hortonworks-hive-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with Hortonworks Hive using JDBC" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + Version: '2012-10-17' + - Statement: + - Action: + - logs:CreateLogGroup + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' + Version: '2012-10-17' + - Statement: + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*' + Version: '2012-10-17' + - Statement: + - Action: + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket + #VPCAccessPolicy allows our connector to run in a VPC so that it can access your data source. + - VPCAccessPolicy: {} + VpcConfig: + SecurityGroupIds: !If [ HasSecurityGroups, !Ref SecurityGroupIds, !Ref "AWS::NoValue" ] + SubnetIds: !If [ HasSubnets, !Ref SubnetIds, !Ref "AWS::NoValue" ] \ No newline at end of file diff --git a/athena-kafka/athena-kafka-package.yaml b/athena-kafka/athena-kafka-package.yaml new file mode 100644 index 0000000000..db99c7f3a5 --- /dev/null +++ b/athena-kafka/athena-kafka-package.yaml @@ -0,0 +1,177 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaKafkaConnector + Description: 'This connector enables Amazon Athena to communicate with Kafka clusters and fetch data.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - kafka + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + AuthType: + Description: 'Authentication details' + Type: String + Default: NO_AUTH + AllowedValues: + - SASL_SSL_SCRAM_SHA512 + - SASL_PLAINTEXT_SCRAM_SHA512 + - SASL_SSL_PLAIN + - SASL_PLAINTEXT_PLAIN + - SSL + - NO_AUTH + KafkaEndpoint: + Description: 'Kafka cluster endpoint' + Type: String + SchemaRegistryUrl: + Description: 'Schema Registry URL. Applicable for Avro/Protobuf Formatted Topics. (syntax. http://:)' + Type: String + Default: "" + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + SecretNamePrefix: + Description: "The secret name within AWS Secrets Manager that contains your aws key and secret Credentials" + Default: "" + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: 'One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: CommaDelimitedList + Default: "" + SubnetIds: + Description: 'One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: CommaDelimitedList + Default: "" + CertificatesS3Reference: + Description: 'The S3 bucket reference where keystore and truststore certificates are uploaded. Applicable for SSL auth' + Default: "" + Type: String + LambdaRoleARN: + Description: "(Optional) A custom role to be used by the Connector lambda" + Default: "" + Type: String + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String + +Conditions: + NotHasLambdaRole: !Equals [!Ref LambdaRoleARN, ""] + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] + HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] + +Resources: + AthenaKafkaConnector: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + secrets_manager_secret: !Ref SecretNamePrefix + certificates_s3_reference: !Ref CertificatesS3Reference + kafka_endpoint: !Ref KafkaEndpoint + schema_registry_url: !Ref SchemaRegistryUrl + auth_type: !Ref AuthType + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.kafka.KafkaCompositeHandler" + CodeUri: "./target/athena-kafka-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with Kafka clusters" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + Role: !If [NotHasLambdaRole, !GetAtt FunctionRole.Arn, !Ref LambdaRoleARN] + VpcConfig: + SecurityGroupIds: !If [ HasSecurityGroups, !Ref SecurityGroupIds, !Ref "AWS::NoValue" ] + SubnetIds: !If [ HasSubnets, !Ref SubnetIds, !Ref "AWS::NoValue"] + + FunctionRole: + Condition: NotHasLambdaRole + Type: AWS::IAM::Role + Properties: + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - "sts:AssumeRole" + + FunctionExecutionPolicy: + Condition: NotHasLambdaRole + Type: "AWS::IAM::Policy" + Properties: + PolicyName: FunctionExecutionPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + - Action: + - logs:CreateLogGroup + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*' + - Action: + - athena:GetQueryExecution + - s3:ListBucket + - s3:GetObject + - s3:GetBucketLocation + - s3:GetObjectVersion + - s3:PutObject + - s3:PutObjectAcl + - s3:GetLifecycleConfiguration + - s3:PutLifecycleConfiguration + - s3:DeleteObject + - kms:Decrypt + - glue:* + Effect: Allow + Resource: '*' + - Action: + - ec2:CreateNetworkInterface + - ec2:DeleteNetworkInterface + - ec2:DescribeNetworkInterfaces + - ec2:DetachNetworkInterface + - autoscaling:CompleteLifecycleAction + Effect: Allow + Resource: '*' + Roles: + - !Ref FunctionRole \ No newline at end of file diff --git a/athena-msk/athena-msk-package.yaml b/athena-msk/athena-msk-package.yaml new file mode 100644 index 0000000000..4adc667169 --- /dev/null +++ b/athena-msk/athena-msk-package.yaml @@ -0,0 +1,173 @@ + +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaMSKConnector + Description: 'This connector enables Amazon Athena to communicate with MSK clusters and fetch data.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - msk + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + AuthType: + Description: 'Authentication details' + Type: String + Default: NO_AUTH + AllowedValues: + - SASL_SSL_AWS_MSK_IAM + - SASL_SSL_SCRAM_SHA512 + - SASL_SSL_PLAIN + - SASL_PLAINTEXT_PLAIN + - SSL + - NO_AUTH + KafkaEndpoint: + Description: 'MSK cluster endpoint' + Type: String + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + SecretNamePrefix: + Description: "The secret name within AWS Secrets Manager that contains your aws key and secret Credentials(Not Required for IAM AUTH)" + Default: "" + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: 'One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: CommaDelimitedList + Default: "" + SubnetIds: + Description: 'One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: CommaDelimitedList + Default: "" + CertificatesS3Reference: + Description: 'The S3 bucket reference where keystore and truststore certificates are uploaded. Applicable for SSL auth' + Default: "" + Type: String + LambdaRoleARN: + Description: "(Must for auth type IAM) A custom role to be used by the Connector lambda" + Default: "" + Type: String + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String + +Conditions: + NotHasLambdaRole: !Equals [!Ref LambdaRoleARN, ""] + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] + HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] + +Resources: + AthenaMSKConnector: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + secrets_manager_secret: !Ref SecretNamePrefix + certificates_s3_reference: !Ref CertificatesS3Reference + kafka_endpoint: !Ref KafkaEndpoint + auth_type: !Ref AuthType + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.msk.AmazonMskCompositeHandler" + CodeUri: "./target/athena-msk-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with MSK clusters" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + Role: !If [NotHasLambdaRole, !GetAtt FunctionRole.Arn, !Ref LambdaRoleARN] + VpcConfig: + SecurityGroupIds: !If [ HasSecurityGroups, !Ref SecurityGroupIds, !Ref "AWS::NoValue" ] + SubnetIds: !If [ HasSubnets, !Ref SubnetIds, !Ref "AWS::NoValue"] + + FunctionRole: + Condition: NotHasLambdaRole + Type: AWS::IAM::Role + Properties: + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - "sts:AssumeRole" + + FunctionExecutionPolicy: + Condition: NotHasLambdaRole + Type: "AWS::IAM::Policy" + Properties: + PolicyName: FunctionExecutionPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + - Action: + - logs:CreateLogGroup + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*' + - Action: + - athena:GetQueryExecution + - s3:ListBucket + - s3:GetObject + - s3:GetBucketLocation + - s3:GetObjectVersion + - s3:PutObject + - s3:PutObjectAcl + - s3:GetLifecycleConfiguration + - s3:PutLifecycleConfiguration + - s3:DeleteObject + - kms:Decrypt + - glue:* + Effect: Allow + Resource: '*' + - Action: + - ec2:CreateNetworkInterface + - ec2:DeleteNetworkInterface + - ec2:DescribeNetworkInterfaces + - ec2:DetachNetworkInterface + - autoscaling:CompleteLifecycleAction + Effect: Allow + Resource: '*' + Roles: + - !Ref FunctionRole diff --git a/athena-mysql/athena-mysql-package.yaml b/athena-mysql/athena-mysql-package.yaml new file mode 100644 index 0000000000..3091da2551 --- /dev/null +++ b/athena-mysql/athena-mysql-package.yaml @@ -0,0 +1,153 @@ + +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaMySQLConnector + Description: 'This connector enables Amazon Athena to communicate with your MySQL instance(s) using JDBC driver.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + DefaultConnectionString: + Description: 'The default connection string is used when catalog is "lambda:${LambdaFunctionName}". Catalog specific Connection Strings can be added later. Format: ${DatabaseType}://${NativeJdbcConnectionString}.' + Type: String + SecretNamePrefix: + Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena MySQL Federation secret names can be prefixed with "AthenaMySQLFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaJdbcFederation*". Parameter value in this case should be "AthenaMySQLFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + LambdaRoleARN: + Description: "(Optional) A custom role to be used by the Connector lambda" + Type: String + Default: "" + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: 'One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: 'List' + SubnetIds: + Description: 'One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: 'List' + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + NotHasLambdaRole: !Equals [!Ref LambdaRoleARN, ""] +Resources: + JdbcConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default: !Ref DefaultConnectionString + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.mysql.MySqlMuxCompositeHandler" + CodeUri: "./target/athena-mysql-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with MySQL using JDBC" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Role: !If [NotHasLambdaRole, !GetAtt FunctionRole.Arn, !Ref LambdaRoleARN] + VpcConfig: + SecurityGroupIds: !Ref SecurityGroupIds + SubnetIds: !Ref SubnetIds + FunctionRole: + Condition: NotHasLambdaRole + Type: AWS::IAM::Role + Properties: + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - "sts:AssumeRole" + FunctionExecutionPolicy: + Condition: NotHasLambdaRole + Type: "AWS::IAM::Policy" + Properties: + Roles: + - !Ref FunctionRole + PolicyName: FunctionExecutionPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + - Action: + - logs:CreateLogGroup + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*' + - Action: + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + - Action: + - ec2:CreateNetworkInterface + - ec2:DeleteNetworkInterface + - ec2:DescribeNetworkInterfaces + - ec2:DetachNetworkInterface + Effect: Allow + Resource: '*' + - Action: + - s3:GetObject + - s3:ListBucket + - s3:GetBucketLocation + - s3:GetObjectVersion + - s3:PutObject + - s3:PutObjectAcl + - s3:GetLifecycleConfiguration + - s3:PutLifecycleConfiguration + - s3:DeleteObject + Effect: Allow + Resource: + - Fn::Sub: + - arn:${AWS::Partition}:s3:::${bucketName} + - bucketName: + Ref: SpillBucket + - Fn::Sub: + - arn:${AWS::Partition}:s3:::${bucketName}/* + - bucketName: + Ref: SpillBucket diff --git a/athena-neptune/athena-neptune-package.yaml b/athena-neptune/athena-neptune-package.yaml new file mode 100644 index 0000000000..c13ed20f89 --- /dev/null +++ b/athena-neptune/athena-neptune-package.yaml @@ -0,0 +1,134 @@ +Transform: 'AWS::Serverless-2016-10-31' + +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaNeptuneConnector + Description: This connector enables Amazon Athena to communicate with your Neptune instance, making your Neptune graph data accessible via SQL. + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: ['athena-federation','athena-neptune','neptune'] + HomePageUrl: https://github.com/awslabs/aws-athena-query-federation + SemanticVersion: 2022.47.1 + SourceCodeUrl: https://github.com/awslabs/aws-athena-query-federation + +Parameters: + NeptuneClusterEndpoint: + Description: 'The Neptune cluster endpoint' + Type: String + NeptunePort: + Description: 'The Neptune port' + Type: String + Default: '8182' + NeptuneClusterResourceID: + Description: 'To find the Neptune cluster resource ID in the Amazon Neptune AWS Management Console, choose the DB cluster that you want. The Resource ID is shown in the Configuration section.' + Type: String + NeptuneGraphType: + Description: 'Type of graph created in Neptune, defaults to PROPERTYGRAPH. Allowed values: PROPERTYGRAPH, RDF' + Type: String + Default: 'PROPERTYGRAPH' + AllowedValues: ["PROPERTYGRAPH", "RDF"] + GlueDatabaseName: + Description: 'Name of the Neptune cluster specific Glue Database that contains schemas of graph vertices' + Type: String + AthenaCatalogName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + Default: 'athena-catalog' + AllowedPattern: ^[a-z0-9-_]{1,64}$ + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-neptune-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: 'One or more SecurityGroup IDs corresponding to the SecurityGroup(s) that should be applied to the Lambda function to allow connectivity to Neptune cluster. (e.g. sg1,sg2,sg3)' + Type: 'List' + SubnetIds: + Description: 'One or more Subnet IDs corresponding to the Subnet(s) that the Lambda function can use to access the Neptune cluster. (e.g. subnet1,subnet2)' + Type: 'List' + IAMEnabled: + Description: 'If set to ''true'' the connector uses Signature Version 4 Signing' + Default: false + Type: String + EnableCaseInsensitiveMatch: + Description: 'If set to ''false'' the connector does a case sensitive match for keys' + Default: true + Type: String + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String + +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + +Resources: + ConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + glue_database_name: !Ref GlueDatabaseName + neptune_endpoint: !Ref NeptuneClusterEndpoint + neptune_port: !Ref NeptunePort + neptune_cluster_res_id: !Ref NeptuneClusterResourceID + iam_enabled: !Ref IAMEnabled + neptune_graphtype: !Ref NeptuneGraphType + SERVICE_REGION: !Ref AWS::Region + enable_caseinsensitivematch: !Ref EnableCaseInsensitiveMatch + FunctionName: !Ref AthenaCatalogName + Handler: "com.amazonaws.athena.connectors.neptune.NeptuneCompositeHandler" + CodeUri: "./target/athena-neptune-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with Neptune, making your Neptune graph data accessible via SQL." + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - glue:GetTableVersions + - glue:GetPartitions + - glue:GetTables + - glue:GetTableVersion + - glue:GetDatabases + - glue:GetTable + - glue:GetPartition + - glue:GetDatabase + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + - Action: + - neptune-db:* + Effect: Allow + #Dynamically construct Neptune Cluster Resource ARN to limit permissions to the specific cluster provided + Resource: !Sub 'arn:${AWS::Partition}:neptune-db:${AWS::Region}:${AWS::AccountId}:${NeptuneClusterResourceID}/*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket + #VPCAccessPolicy allows our connector to run in a VPC so that it can access your data source. + - VPCAccessPolicy: {} + VpcConfig: + SecurityGroupIds: !Ref SecurityGroupIds + SubnetIds: !Ref SubnetIds \ No newline at end of file diff --git a/athena-oracle/athena-oracle-package.yaml b/athena-oracle/athena-oracle-package.yaml new file mode 100644 index 0000000000..76d3b4dca1 --- /dev/null +++ b/athena-oracle/athena-oracle-package.yaml @@ -0,0 +1,158 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaOracleConnector + Description: 'This connector enables Amazon Athena to communicate with your ORACLE instance(s) using JDBC driver.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + DefaultConnectionString: + Description: 'The default connection string is used when catalog is "lambda:${LambdaFunctionName}". Catalog specific Connection Strings can be added later. Format: ${DatabaseType}://${NativeJdbcConnectionString}.' + Type: String + SecretNamePrefix: + Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena JDBC Federation secret names can be prefixed with "AthenaJdbcFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaJdbcFederation*". Parameter value in this case should be "AthenaJdbcFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' + Type: String + IsFIPSEnabled: + AllowedValues: + - true + - false + Default: false + Description: "If oracle rds is FIPS compliance select true, [true, false] (default is false)" + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + LambdaRoleARN: + Description: "(Optional) A custom role to be used by the Connector lambda" + Type: String + Default: "" + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: '(Optional) One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: CommaDelimitedList + Default: "" + SubnetIds: + Description: '(Optional) One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: CommaDelimitedList + Default: "" + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + NotHasLambdaRole: !Equals [!Ref LambdaRoleARN, ""] + HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] + HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] +Resources: + JdbcConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default: !Ref DefaultConnectionString + is_FIPS_Enabled: !Ref IsFIPSEnabled + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.oracle.OracleMuxCompositeHandler" + CodeUri: "./target/athena-oracle-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with ORACLE using JDBC" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Role: !If [NotHasLambdaRole, !GetAtt FunctionRole.Arn, !Ref LambdaRoleARN] + VpcConfig: + SecurityGroupIds: !If [ HasSecurityGroups, !Ref SecurityGroupIds, !Ref "AWS::NoValue" ] + SubnetIds: !If [ HasSubnets, !Ref SubnetIds, !Ref "AWS::NoValue" ] + + FunctionRole: + Condition: NotHasLambdaRole + Type: AWS::IAM::Role + Properties: + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - "sts:AssumeRole" + FunctionExecutionPolicy: + Condition: NotHasLambdaRole + Type: "AWS::IAM::Policy" + Properties: + Roles: + - !Ref FunctionRole + PolicyName: FunctionExecutionPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + - Action: + - logs:CreateLogGroup + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*' + - Action: + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + - Action: + - s3:GetObject + - s3:ListBucket + - s3:GetBucketLocation + - s3:GetObjectVersion + - s3:PutObject + - s3:PutObjectAcl + - s3:GetLifecycleConfiguration + - s3:PutLifecycleConfiguration + - s3:DeleteObject + Effect: Allow + Resource: + - Fn::Sub: + - arn:${AWS::Partition}:s3:::${bucketName} + - bucketName: + Ref: SpillBucket + - Fn::Sub: + - arn:${AWS::Partition}:s3:::${bucketName}/* + - bucketName: + Ref: SpillBucket \ No newline at end of file diff --git a/athena-postgresql/athena-postgresql-package.yaml b/athena-postgresql/athena-postgresql-package.yaml new file mode 100644 index 0000000000..695f0759ca --- /dev/null +++ b/athena-postgresql/athena-postgresql-package.yaml @@ -0,0 +1,165 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaPostgreSQLConnector + Description: 'This connector enables Amazon Athena to communicate with your PostgreSQL instance(s) using JDBC driver.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + DefaultConnectionString: + Description: 'The default connection string is used when catalog is "lambda:${LambdaFunctionName}". Catalog specific Connection Strings can be added later. Format: ${DatabaseType}://${NativeJdbcConnectionString}.' + Type: String + SecretNamePrefix: + Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena PostgreSQL Federation secret names can be prefixed with "AthenaPostgreSQLFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaJdbcFederation*". Parameter value in this case should be "AthenaPostgreSQLFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + LambdaRoleARN: + Description: "(Optional) A custom role to be used by the Connector lambda" + Type: String + Default: "" + SecurityGroupIds: + Description: 'One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: 'List' + SubnetIds: + Description: 'One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: 'List' + CompositeHandler: + Description: 'Use "PostGreSqlMuxCompositeHandler" to access multiple postgres instances and "PostGreSqlCompositeHandler" to access single instance using DefaultConnectionString' + Type: String + Default: "PostGreSqlMuxCompositeHandler" + AllowedValues : ["PostGreSqlMuxCompositeHandler", "PostGreSqlCompositeHandler"] + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String + DefaultScale: + Description: "(Optional) Default value for scale of type Numeric, representing the decimal digits in the fractional part, to the right of the decimal point." + Default: 0 + Type: Number + +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + NotHasLambdaRole: !Equals [!Ref LambdaRoleARN, ""] +Resources: + JdbcConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default: !Ref DefaultConnectionString + default_scale: !Ref DefaultScale + FunctionName: !Ref LambdaFunctionName + Handler: !Sub "com.amazonaws.athena.connectors.postgresql.${CompositeHandler}" + CodeUri: "./target/athena-postgresql-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with PostgreSQL using JDBC" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Role: !If [NotHasLambdaRole, !GetAtt FunctionRole.Arn, !Ref LambdaRoleARN] + VpcConfig: + SecurityGroupIds: !Ref SecurityGroupIds + SubnetIds: !Ref SubnetIds + + FunctionRole: + Condition: NotHasLambdaRole + Type: AWS::IAM::Role + Properties: + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - "sts:AssumeRole" + + FunctionExecutionPolicy: + Condition: NotHasLambdaRole + Type: "AWS::IAM::Policy" + Properties: + Roles: + - !Ref FunctionRole + PolicyName: FunctionExecutionPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + - Action: + - logs:CreateLogGroup + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*' + - Action: + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + - Action: + - ec2:CreateNetworkInterface + - ec2:DeleteNetworkInterface + - ec2:DescribeNetworkInterfaces + - ec2:DetachNetworkInterface + Effect: Allow + Resource: '*' + - Action: + - s3:GetObject + - s3:ListBucket + - s3:GetBucketLocation + - s3:GetObjectVersion + - s3:PutObject + - s3:PutObjectAcl + - s3:GetLifecycleConfiguration + - s3:PutLifecycleConfiguration + - s3:DeleteObject + Effect: Allow + Resource: + - Fn::Sub: + - arn:${AWS::Partition}:s3:::${bucketName} + - bucketName: + Ref: SpillBucket + - Fn::Sub: + - arn:${AWS::Partition}:s3:::${bucketName}/* + - bucketName: + Ref: SpillBucket \ No newline at end of file diff --git a/athena-redis/athena-redis-package.yaml b/athena-redis/athena-redis-package.yaml new file mode 100644 index 0000000000..85ffd47a79 --- /dev/null +++ b/athena-redis/athena-redis-package.yaml @@ -0,0 +1,120 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaRedisConnector + Description: 'This connector enables Amazon Athena to communicate with your Redis instance(s), making your Redis data accessible via SQL.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + AthenaCatalogName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: "WARNING: If set to 'true' encryption for spilled data is disabled." + Default: 'false' + Type: String + SecurityGroupIds: + Description: 'One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: 'List' + SubnetIds: + Description: 'One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: 'List' + SecretNameOrPrefix: + Description: 'The name or prefix of a set of names within Secrets Manager that this function should have access to. (e.g. redis-*).' + Type: String + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String + QPTConnectionEndpoint: + Description: "(Optional) The hostname:port:password of the Redis server that contains data for this table optionally using SecretsManager (e.g. ${secret_name}). Used for Query Pass Through queries only." + Default: '' + Type: String + QPTConnectionSSL: + Description: "(Optional) When True, creates a Redis connection that uses SSL/TLS. Used for Query Pass Through queries only." + Default: 'false' + Type: String + QPTConnectionCluster: + Description: "(Optional) When True, enables support for clustered Redis instances. Used for Query Pass Through queries only." + Default: 'false' + Type: String + QPTConnectionDBNumber: + Description: "(Optional) Set this number (for example 1, 2, or 3) to read from a non-default Redis database. Used for Query Pass Through queries only." + Default: 0 + Type: Number +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] +Resources: + ConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + qpt_endpoint: !Ref QPTConnectionEndpoint + qpt_ssl: !Ref QPTConnectionSSL + qpt_cluster: !Ref QPTConnectionCluster + qpt_db_number: !Ref QPTConnectionDBNumber + FunctionName: !Ref AthenaCatalogName + Handler: "com.amazonaws.athena.connectors.redis.RedisCompositeHandler" + CodeUri: "./target/athena-redis-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with Redis, making your Redis data accessible via SQL" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:*:*:secret:${SecretNameOrPrefix}' + Version: '2012-10-17' + - Statement: + - Action: + - glue:GetTableVersions + - glue:GetPartitions + - glue:GetTables + - glue:GetTableVersion + - glue:GetDatabases + - glue:GetTable + - glue:GetPartition + - glue:GetDatabase + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket + #VPCAccessPolicy allows our connector to run in a VPC so that it can access your data source. + - VPCAccessPolicy: {} + VpcConfig: + SecurityGroupIds: !Ref SecurityGroupIds + SubnetIds: !Ref SubnetIds \ No newline at end of file diff --git a/athena-redshift/athena-redshift-package.yaml b/athena-redshift/athena-redshift-package.yaml new file mode 100644 index 0000000000..7a65432d4b --- /dev/null +++ b/athena-redshift/athena-redshift-package.yaml @@ -0,0 +1,167 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaRedshiftConnector + Description: 'This connector enables Amazon Athena to communicate with your Redshift instance(s) using JDBC driver.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + DefaultConnectionString: + Description: 'The default connection string is used when catalog is "lambda:${LambdaFunctionName}". Catalog specific Connection Strings can be added later. Format: ${DatabaseType}://${NativeJdbcConnectionString}.' + Type: String + SecretNamePrefix: + Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena Redshift Federation secret names can be prefixed with "AthenaRedshiftFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaRedshiftFederation*". Parameter value in this case should be "AthenaRedshiftFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: 'One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: 'List' + SubnetIds: + Description: 'One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: 'List' + KMSKeyId: + Description: "(Optional) By default any data that is spilled to S3 is encrypted using AES-GCM and a randomly generated key. Setting a KMS Key ID allows your Lambda function to use KMS for key generation for a stronger source of encryption keys." + Type: String + Default: "" + LambdaRole: + Description: "(Optional) A custom IAM role ARN to be used by the Connector lambda" + Type: String + Default: "" + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasKMSKeyId: !Not [!Equals [!Ref KMSKeyId, ""]] + NotHasLambdaRole: !Equals [!Ref LambdaRole, ""] + NotHasLambdaRoleAndHasKMSKeyId: !And + - !Condition NotHasLambdaRole + - !Condition HasKMSKeyId + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] +Resources: + JdbcConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default: !Ref DefaultConnectionString + kms_key_id: !If [HasKMSKeyId, !Ref KMSKeyId, !Ref "AWS::NoValue"] + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.redshift.RedshiftMuxCompositeHandler" + CodeUri: "./target/athena-redshift-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with Redshift using JDBC" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + Role: !If [NotHasLambdaRole, !GetAtt FunctionRole.Arn, !Ref LambdaRole] + VpcConfig: + SecurityGroupIds: !Ref SecurityGroupIds + SubnetIds: !Ref SubnetIds + + FunctionRole: + Condition: NotHasLambdaRole + Type: AWS::IAM::Role + Properties: + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - "sts:AssumeRole" + + + FunctionExecutionPolicy: + Condition: NotHasLambdaRole + Type: "AWS::IAM::Policy" + Properties: + PolicyName: FunctionExecutionPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - secretsmanager:GetSecretValue + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + - Effect: Allow + Action: + - athena:GetQueryExecution + Resource: '*' + - Effect: Allow + Action: + - s3:GetObject + - s3:ListBucket + - s3:GetBucketLocation + - s3:GetObjectVersion + - s3:PutObject + - s3:PutObjectAcl + - s3:GetLifecycleConfiguration + - s3:PutLifecycleConfiguration + - s3:DeleteObject + Resource: + - Fn::Sub: + - arn:${AWS::Partition}:s3:::${bucketName} + - bucketName: + Ref: SpillBucket + - Fn::Sub: + - arn:${AWS::Partition}:s3:::${bucketName}/* + - bucketName: + Ref: SpillBucket + Roles: + - !Ref FunctionRole + + FunctionKMSPolicy: + Condition: NotHasLambdaRoleAndHasKMSKeyId + Type: "AWS::IAM::Policy" + Properties: + PolicyName: FunctionKMSPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - kms:GenerateRandom + Resource: '*' + - Effect: Allow + Action: + - kms:GenerateDataKey + Resource: !Sub "arn:${AWS::Partition}:kms:${AWS::Region}:${AWS::AccountId}:key/${KMSKeyId}" + Roles: + - !Ref FunctionRole \ No newline at end of file diff --git a/athena-saphana/athena-saphana-package.yaml b/athena-saphana/athena-saphana-package.yaml new file mode 100644 index 0000000000..d85a95b3eb --- /dev/null +++ b/athena-saphana/athena-saphana-package.yaml @@ -0,0 +1,113 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaSaphanaConnector + Description: 'This connector enables Amazon Athena to communicate with your Teradata instance(s) using JDBC driver.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + DefaultConnectionString: + Description: 'The default connection string is used when catalog is "lambda:${LambdaFunctionName}". Catalog specific Connection Strings can be added later. Format: ${DatabaseType}://${NativeJdbcConnectionString}.' + Type: String + SecretNamePrefix: + Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena JDBC Federation secret names can be prefixed with "AthenaJdbcFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaJdbcFederation*". Parameter value in this case should be "AthenaJdbcFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: '(Optional) One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: CommaDelimitedList + Default: "" + SubnetIds: + Description: '(Optional) One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: CommaDelimitedList + Default: "" + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] + HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] +Resources: + JdbcConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default: !Ref DefaultConnectionString + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.saphana.SaphanaMuxCompositeHandler" + CodeUri: "./target/athena-saphana.zip" + Description: "Enables Amazon Athena to communicate with Teradata using JDBC" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + Version: '2012-10-17' + - Statement: + - Action: + - logs:CreateLogGroup + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' + Version: '2012-10-17' + - Statement: + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*' + Version: '2012-10-17' + - Statement: + - Action: + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket + #VPCAccessPolicy allows our connector to run in a VPC so that it can access your data source. + - VPCAccessPolicy: {} + VpcConfig: + SecurityGroupIds: !If [ HasSecurityGroups, !Ref SecurityGroupIds, !Ref "AWS::NoValue" ] + SubnetIds: !If [ HasSubnets, !Ref SubnetIds, !Ref "AWS::NoValue" ] \ No newline at end of file diff --git a/athena-sqlserver/athena-sqlserver-package.yaml b/athena-sqlserver/athena-sqlserver-package.yaml new file mode 100644 index 0000000000..a560854c7f --- /dev/null +++ b/athena-sqlserver/athena-sqlserver-package.yaml @@ -0,0 +1,158 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaSqlServerConnector + Description: 'This connector enables Amazon Athena to communicate with SQL Server using JDBC driver.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - sqlserver + - athena-federation + - jdbc + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + DefaultConnectionString: + Description: 'The default connection string is used when catalog is "lambda:${LambdaFunctionName}". Catalog specific Connection Strings can be added later. Format: ${DatabaseType}://${NativeJdbcConnectionString}.' + Type: String + SecretNamePrefix: + Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena JDBC Federation secret names can be prefixed with "AthenaJdbcFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaJdbcFederation*". Parameter value in this case should be "AthenaJdbcFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + LambdaRoleARN: + Description: "(Optional) A custom role to be used by the Connector lambda" + Type: String + Default: "" + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: '(Optional) One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: CommaDelimitedList + Default: "" + SubnetIds: + Description: '(Optional) One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: CommaDelimitedList + Default: "" + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + NotHasLambdaRole: !Equals [!Ref LambdaRoleARN, ""] + HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] + HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] +Resources: + JdbcConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default: !Ref DefaultConnectionString + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.sqlserver.SqlServerMuxCompositeHandler" + CodeUri: "./target/athena-sqlserver-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with SQLSERVER using JDBC" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Role: !If [NotHasLambdaRole, !GetAtt FunctionRole.Arn, !Ref LambdaRoleARN] + VpcConfig: + SecurityGroupIds: !If [ HasSecurityGroups, !Ref SecurityGroupIds, !Ref "AWS::NoValue" ] + SubnetIds: !If [ HasSubnets, !Ref SubnetIds, !Ref "AWS::NoValue" ] + FunctionRole: + Condition: NotHasLambdaRole + Type: AWS::IAM::Role + Properties: + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - "sts:AssumeRole" + FunctionExecutionPolicy: + Condition: NotHasLambdaRole + Type: "AWS::IAM::Policy" + Properties: + Roles: + - !Ref FunctionRole + PolicyName: FunctionExecutionPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + - Action: + - logs:CreateLogGroup + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*' + - Action: + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + - Action: + - ec2:CreateNetworkInterface + - ec2:DeleteNetworkInterface + - ec2:DescribeNetworkInterfaces + - ec2:DetachNetworkInterface + Effect: Allow + Resource: '*' + - Action: + - s3:GetObject + - s3:ListBucket + - s3:GetBucketLocation + - s3:GetObjectVersion + - s3:PutObject + - s3:PutObjectAcl + - s3:GetLifecycleConfiguration + - s3:PutLifecycleConfiguration + - s3:DeleteObject + Effect: Allow + Resource: + - Fn::Sub: + - arn:${AWS::Partition}:s3:::${bucketName} + - bucketName: + Ref: SpillBucket + - Fn::Sub: + - arn:${AWS::Partition}:s3:::${bucketName}/* + - bucketName: + Ref: SpillBucket \ No newline at end of file diff --git a/athena-synapse/athena-synapse-package.yaml b/athena-synapse/athena-synapse-package.yaml new file mode 100644 index 0000000000..ccfed7728b --- /dev/null +++ b/athena-synapse/athena-synapse-package.yaml @@ -0,0 +1,147 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaSynapseConnector + Description: 'This connector enables Amazon Athena to communicate with Synapse using JDBC driver.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - synapse + - athena-federation + - jdbc + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + DefaultConnectionString: + Description: 'The default connection string is used when catalog is "lambda:${LambdaFunctionName}". Catalog specific Connection Strings can be added later. Format: ${DatabaseType}://${NativeJdbcConnectionString}.' + Type: String + SecretNamePrefix: + Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena JDBC Federation secret names can be prefixed with "AthenaJdbcFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaJdbcFederation*". Parameter value in this case should be "AthenaJdbcFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + LambdaRoleARN: + Description: "(Optional) A custom role to be used by the Connector lambda" + Default: "" + Type: String + PermissionsBoundaryARN: + Description: "(Optional) A custom Permission Boundary to be used by the Connector lambda" + Default: "" + Type: String + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: '(Optional) One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: CommaDelimitedList + Default: "" + SubnetIds: + Description: '(Optional) One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: CommaDelimitedList + Default: "" + +Conditions: + NotHasLambdaRole: !Equals [!Ref LambdaRoleARN, ""] + HasPermissionsBoundary: !Not [!Equals [!Ref PermissionsBoundaryARN, ""]] + HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] + HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] + +Resources: + JdbcConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default: !Ref DefaultConnectionString + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.synapse.SynapseMuxCompositeHandler" + CodeUri: "./target/athena-synapse-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with SYNPASE using JDBC" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + Role: !If [NotHasLambdaRole, !GetAtt FunctionRole.Arn, !Ref LambdaRoleARN] + VpcConfig: + SecurityGroupIds: !If [ HasSecurityGroups, !Ref SecurityGroupIds, !Ref "AWS::NoValue" ] + SubnetIds: !If [ HasSubnets, !Ref SubnetIds, !Ref "AWS::NoValue" ] + + FunctionRole: + Condition: NotHasLambdaRole + Type: AWS::IAM::Role + Properties: + PermissionsBoundary: + Fn::If: + - HasPermissionsBoundary + - Ref: PermissionsBoundaryARN + - Ref: AWS::NoValue + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - "sts:AssumeRole" + + FunctionExecutionPolicy: + Condition: NotHasLambdaRole + Type: "AWS::IAM::Policy" + Properties: + PolicyName: FunctionExecutionPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + - Action: + - logs:CreateLogGroup + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*' + - Action: + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + - Action: + - ec2:CreateNetworkInterface + - ec2:DeleteNetworkInterface + - ec2:DescribeNetworkInterfaces + - ec2:DetachNetworkInterface + - autoscaling:CompleteLifecycleAction + Effect: Allow + Resource: '*' + Roles: + - !Ref FunctionRole \ No newline at end of file diff --git a/athena-teradata/athena-teradata-package.yaml b/athena-teradata/athena-teradata-package.yaml new file mode 100644 index 0000000000..3e1431c6c0 --- /dev/null +++ b/athena-teradata/athena-teradata-package.yaml @@ -0,0 +1,123 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaTeradataConnector + Description: 'This connector enables Amazon Athena to communicate with your Teradata instance(s) using JDBC driver.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + LambdaFunctionName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + DefaultConnectionString: + Description: 'The default connection string is used when catalog is "lambda:${LambdaFunctionName}". Catalog specific Connection Strings can be added later. Format: ${DatabaseType}://${NativeJdbcConnectionString}.' + Type: String + SecretNamePrefix: + Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena JDBC Federation secret names can be prefixed with "AthenaJdbcFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaJdbcFederation*". Parameter value in this case should be "AthenaJdbcFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' + Type: String + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaJDBCLayername: + Description: 'Lambda JDBC layer Name. Must be ARN of layer' + Type: String + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: 'If set to ''false'' data spilled to S3 is encrypted with AES GCM' + Default: 'false' + Type: String + SecurityGroupIds: + Description: '(Optional) One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' + Type: CommaDelimitedList + Default: "" + SubnetIds: + Description: '(Optional) One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: CommaDelimitedList + Default: "" + PartitionCount: + Description: 'Partition Count Limit' + Type: Number + Default: 500 + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] + HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] +Resources: + JdbcConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + default: !Ref DefaultConnectionString + partitioncount: !Ref PartitionCount + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.teradata.TeradataMuxCompositeHandler" + Layers: + - !Ref LambdaJDBCLayername + CodeUri: "./target/athena-teradata-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with Teradata using JDBC" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretNamePrefix}*' + Version: '2012-10-17' + - Statement: + - Action: + - logs:CreateLogGroup + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' + Version: '2012-10-17' + - Statement: + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*' + Version: '2012-10-17' + - Statement: + - Action: + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket + #VPCAccessPolicy allows our connector to run in a VPC so that it can access your data source. + - VPCAccessPolicy: {} + VpcConfig: + SecurityGroupIds: !If [ HasSecurityGroups, !Ref SecurityGroupIds, !Ref "AWS::NoValue" ] + SubnetIds: !If [ HasSubnets, !Ref SubnetIds, !Ref "AWS::NoValue" ] \ No newline at end of file diff --git a/athena-timestream/athena-timestream-package.yaml b/athena-timestream/athena-timestream-package.yaml new file mode 100644 index 0000000000..3ee8788678 --- /dev/null +++ b/athena-timestream/athena-timestream-package.yaml @@ -0,0 +1,83 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaTimestreamConnector + Description: 'This connector enables Amazon Athena to communicate with Amazon Timestream, making your time series data accessible from Athena.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + AthenaCatalogName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: "WARNING: If set to 'true' encryption for spilled data is disabled." + Default: 'false' + Type: String + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] +Resources: + ConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + FunctionName: !Ref AthenaCatalogName + Handler: "com.amazonaws.athena.connectors.timestream.TimestreamCompositeHandler" + CodeUri: "./target/athena-timestream-2022.47.1.jar" + Description: "Enables Amazon Athena to communicate with Amazon Timestream, making your time series data accessible from Athena." + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - glue:GetTableVersions + - glue:GetPartitions + - glue:GetTables + - glue:GetTableVersion + - glue:GetDatabases + - glue:GetTable + - glue:GetPartition + - glue:GetDatabase + - athena:GetQueryExecution + - timestream:Describe* + - timestream:List* + - timestream:Select* + Effect: Allow + Resource: '*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket \ No newline at end of file diff --git a/athena-tpcds/athena-tpcds-package.yaml b/athena-tpcds/athena-tpcds-package.yaml new file mode 100644 index 0000000000..d2897ac5b8 --- /dev/null +++ b/athena-tpcds/athena-tpcds-package.yaml @@ -0,0 +1,73 @@ + +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaTPCDSConnector + Description: 'This connector enables Amazon Athena to communicate with a randomly generated TPC-DS data source.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + AthenaCatalogName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: "WARNING: If set to 'true' encryption for spilled data is disabled." + Default: 'false' + Type: String + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] +Resources: + ConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + FunctionName: !Ref AthenaCatalogName + Handler: "com.amazonaws.athena.connectors.tpcds.TPCDSCompositeHandler" + CodeUri: "./target/athena-tpcds-2022.47.1.jar" + Description: "This connector enables Amazon Athena to communicate with a randomly generated TPC-DS data source." + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + Version: '2012-10-17' + #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy + #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files. + - S3CrudPolicy: + BucketName: !Ref SpillBucket diff --git a/athena-udfs/athena-udfs-package.yaml b/athena-udfs/athena-udfs-package.yaml new file mode 100644 index 0000000000..3fdea75381 --- /dev/null +++ b/athena-udfs/athena-udfs-package.yaml @@ -0,0 +1,55 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaUserDefinedFunctionsV2 + Description: 'This connector enables Amazon Athena to leverage common UDFs made available via Lambda.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2022.47.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + LambdaFunctionName: + Description: 'The name you will give to Lambda function which executes your UDFs. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + LambdaTimeout: + Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' + Default: 900 + Type: Number + LambdaMemory: + Description: 'Lambda memory in MB (min 128 - 3008 max).' + Default: 3008 + Type: Number + SecretName: + Description: 'The name or prefix of a set of names within Secrets Manager that this function should have access to. (e.g. database-*).' + Type: String + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] +Resources: + ConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + FunctionName: !Ref LambdaFunctionName + Handler: "com.amazonaws.athena.connectors.udfs.AthenaUDFHandler" + CodeUri: "./target/athena-udfs-2022.47.1.jar" + Description: "This connector enables Amazon Athena to leverage common UDFs made available via Lambda." + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:*:*:secret:${SecretName}' + Version: '2012-10-17' \ No newline at end of file diff --git a/athena-vertica/athena-vertica-package.yaml b/athena-vertica/athena-vertica-package.yaml new file mode 100644 index 0000000000..65cb959214 --- /dev/null +++ b/athena-vertica/athena-vertica-package.yaml @@ -0,0 +1,129 @@ +Transform: 'AWS::Serverless-2016-10-31' + +Metadata: + AWS::ServerlessRepo::Application: + Name: AthenaVerticaConnector + Description: 'This connector enables Amazon Athena to communicate with Vertica' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: ['athena-federation'] + HomePageUrl: https://github.com/awslabs/aws-athena-query-federation + SemanticVersion: 2022.47.1 + SourceCodeUrl: https://github.com/awslabs/aws-athena-query-federation + +# Parameters are CloudFormation features to pass input +# to your template when you create a stack +Parameters: + AthenaCatalogName: + Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' + Type: String + AllowedPattern: ^[a-z0-9-_]{1,64}$ + SpillBucket: + Description: 'The name of the bucket where this function can spill data.' + Type: String + VerticaExportBucket: + Description: "The bucket where the Vertica Query results will be exported." + Type: String + SpillPrefix: + Description: 'The prefix within SpillBucket where this function can spill data.' + Type: String + Default: athena-spill + LambdaTimeout: + Description: "Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)" + Default: 900 + Type: Number + LambdaMemory: + Description: "Lambda memory in MB (min 128 - 3008 max)." + Default: 3008 + Type: Number + DisableSpillEncryption: + Description: "WARNING: If set to 'true' encryption for spilled data is disabled." + Default: "false" + Type: String + VpcId: + Description: 'VPC ID' + Type: 'AWS::EC2::VPC::Id' + SubnetIds: + Description: 'One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' + Type: 'List' + SecretNameOrPrefix: + Description: 'The name or prefix of a set of names within Secrets Manager that this function should have access to. (e.g. vertica-*).' + Type: String + Default: "vertica-*" + VerticaConnectionString: + Description: 'The Vertica connection details to use by default if not catalog specific connection is defined and optionally using SecretsManager (e.g. ${secret_name}).' + Type: String + Default: "vertica://jdbc:vertica://:/?user=${vertica-username}&password=${vertica-password}" + PermissionsBoundaryARN: + Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" + Default: '' + Type: String + +Conditions: + HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + +Resources: + LambdaSecurityGroup: + Type: 'AWS::EC2::SecurityGroup' + Properties: + GroupDescription: 'Athena Vertica Connector Lambda VPC Security Group' + VpcId: !Ref VpcId + ConnectorConfig: + Type: 'AWS::Serverless::Function' + Properties: + Environment: + Variables: + disable_spill_encryption: !Ref DisableSpillEncryption + spill_bucket: !Ref SpillBucket + spill_prefix: !Ref SpillPrefix + export_bucket: !Ref VerticaExportBucket + default: !Ref VerticaConnectionString + + FunctionName: !Sub "${AthenaCatalogName}" + Handler: "com.amazonaws.athena.connectors.vertica.VerticaCompositeHandler" + CodeUri: "./target/athena-vertica-2022.47.1.jar" + Description: "Amazon Athena Vertica Connector" + Runtime: java11 + Timeout: !Ref LambdaTimeout + MemorySize: !Ref LambdaMemory + PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] + Policies: + - Statement: + - Action: + - athena:GetQueryExecution + Effect: Allow + Resource: '*' + Version: '2012-10-17' + - Statement: + - Action: + - s3:ListBucket + Effect: Allow + Resource: + - !Sub 'arn:${AWS::Partition}:s3:::${VerticaExportBucket}' + - !Sub 'arn:${AWS::Partition}:s3:::${SpillBucket}' + Version: '2012-10-17' + - Statement: + - Action: + - secretsmanager:GetSecretValue + Effect: Allow + Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:*:*:secret:${SecretNameOrPrefix}' + - S3ReadPolicy: + BucketName: + Ref: SpillBucket + - S3WritePolicy: + BucketName: + Ref: SpillBucket + - S3ReadPolicy: + BucketName: + Ref: VerticaExportBucket + - S3WritePolicy: + BucketName: + Ref: VerticaExportBucket + #VPCAccessPolicy allows our connector to run in a VPC so that it can access your data source. + VpcConfig: + SecurityGroupIds: + #SecurityGroup that should be applied to the Lambda function + - !Ref LambdaSecurityGroup + SubnetIds: !Ref SubnetIds \ No newline at end of file From 3b67035632413ff7e5319dc1f177bc497121f880 Mon Sep 17 00:00:00 2001 From: Aimery Methena Date: Fri, 7 Feb 2025 10:52:34 -0500 Subject: [PATCH 2/2] fix errors in templates --- ...yaml => athena-elasticsearch-package.yaml} | 0 athena-gcs/athena-gcs-package.yaml | 101 -------------- athena-teradata/athena-teradata-package.yaml | 5 - athena-vertica/athena-vertica-package.yaml | 129 ------------------ 4 files changed, 235 deletions(-) rename athena-elasticsearch/{athena-elasticesearch-package.yaml => athena-elasticsearch-package.yaml} (100%) delete mode 100644 athena-gcs/athena-gcs-package.yaml delete mode 100644 athena-vertica/athena-vertica-package.yaml diff --git a/athena-elasticsearch/athena-elasticesearch-package.yaml b/athena-elasticsearch/athena-elasticsearch-package.yaml similarity index 100% rename from athena-elasticsearch/athena-elasticesearch-package.yaml rename to athena-elasticsearch/athena-elasticsearch-package.yaml diff --git a/athena-gcs/athena-gcs-package.yaml b/athena-gcs/athena-gcs-package.yaml deleted file mode 100644 index 621998ba81..0000000000 --- a/athena-gcs/athena-gcs-package.yaml +++ /dev/null @@ -1,101 +0,0 @@ -Transform: 'AWS::Serverless-2016-10-31' - -Metadata: - AWS::ServerlessRepo::Application: - Name: AthenaGCSConnector - Description: 'This connector enables Amazon Athena to communicate with Google Cloud Storage (GCS) and fetch data from Parquet file format' - Author: 'default author' - SpdxLicenseId: Apache-2.0 - LicenseUrl: LICENSE.txt - ReadmeUrl: README.md - Labels: ['athena-federation', 'GCS', 'Google-Cloud-Storage', 'parquet', 'csv'] - HomePageUrl: https://github.com/awslabs/aws-athena-query-federation - SemanticVersion: 2022.47.1 - SourceCodeUrl: https://github.com/awslabs/aws-athena-query-federation - -Parameters: - LambdaFunctionName: - Description: 'The name you will give to this catalog is a Lambda name for Athena. Athena will use this name as the function name. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' - Type: String - AllowedPattern: ^[a-z0-9-_]{1,64}$ - SpillBucket: - Description: 'The name of the bucket where this function can spill data.' - Type: String - SpillPrefix: - Description: 'The prefix within SpillBucket where this function can spill data.' - Type: String - Default: athena-spill - LambdaTimeout: - Description: "Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)" - Default: 900 - Type: Number - LambdaMemory: - Description: "Lambda memory in MB (min 128 - 3008 max)." - Default: 3008 - Type: Number - DisableSpillEncryption: - Description: "WARNING: If set to 'true' encryption for spilled data is disabled." - Default: "false" - Type: String - GCSSecretName: - Description: 'Secret key name in the AWS Secrets Manager.' - Type: String - PermissionsBoundaryARN: - Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" - Default: '' - Type: String - -Conditions: - HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] - -Resources: - AthenaGCSConnector: - Type: 'AWS::Serverless::Function' - Properties: - Environment: - Variables: - disable_spill_encryption: !Ref DisableSpillEncryption - spill_bucket: !Ref SpillBucket - spill_prefix: !Ref SpillPrefix - secret_manager_gcp_creds_name: !Ref GCSSecretName - FunctionName: !Ref LambdaFunctionName - Handler: "com.amazonaws.athena.connectors.gcs.GcsCompositeHandler" - CodeUri: "./target/athena-gcs.zip" - Description: "Amazon Athena GCS Connector" - Runtime: java11 - Timeout: !Ref LambdaTimeout - MemorySize: !Ref LambdaMemory - PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] - Policies: - - Statement: - - Action: - - athena:GetQueryExecution - - glue:GetTableVersions - - glue:GetPartitions - - glue:GetTables - - glue:GetTableVersion - - glue:GetDatabases - - glue:GetTable - - glue:GetPartition - - glue:GetDatabase - Effect: Allow - Resource: '*' - Version: '2012-10-17' - - Statement: - - Action: - - s3:ListBucket - Effect: Allow - Resource: - - !Sub 'arn:${AWS::Partition}:s3:::${SpillBucket}' - Version: '2012-10-17' - - Statement: - - Action: - - secretsmanager:GetSecretValue - Effect: Allow - Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:*:*:secret:${GCSSecretName}*' - - S3ReadPolicy: - BucketName: - Ref: SpillBucket - - S3WritePolicy: - BucketName: - Ref: SpillBucket \ No newline at end of file diff --git a/athena-teradata/athena-teradata-package.yaml b/athena-teradata/athena-teradata-package.yaml index 3e1431c6c0..70cfa6151d 100644 --- a/athena-teradata/athena-teradata-package.yaml +++ b/athena-teradata/athena-teradata-package.yaml @@ -30,9 +30,6 @@ Parameters: Description: 'The prefix within SpillBucket where this function can spill data.' Type: String Default: athena-spill - LambdaJDBCLayername: - Description: 'Lambda JDBC layer Name. Must be ARN of layer' - Type: String LambdaTimeout: Description: 'Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)' Default: 900 @@ -78,8 +75,6 @@ Resources: partitioncount: !Ref PartitionCount FunctionName: !Ref LambdaFunctionName Handler: "com.amazonaws.athena.connectors.teradata.TeradataMuxCompositeHandler" - Layers: - - !Ref LambdaJDBCLayername CodeUri: "./target/athena-teradata-2022.47.1.jar" Description: "Enables Amazon Athena to communicate with Teradata using JDBC" Runtime: java11 diff --git a/athena-vertica/athena-vertica-package.yaml b/athena-vertica/athena-vertica-package.yaml deleted file mode 100644 index 65cb959214..0000000000 --- a/athena-vertica/athena-vertica-package.yaml +++ /dev/null @@ -1,129 +0,0 @@ -Transform: 'AWS::Serverless-2016-10-31' - -Metadata: - AWS::ServerlessRepo::Application: - Name: AthenaVerticaConnector - Description: 'This connector enables Amazon Athena to communicate with Vertica' - Author: 'default author' - SpdxLicenseId: Apache-2.0 - LicenseUrl: LICENSE.txt - ReadmeUrl: README.md - Labels: ['athena-federation'] - HomePageUrl: https://github.com/awslabs/aws-athena-query-federation - SemanticVersion: 2022.47.1 - SourceCodeUrl: https://github.com/awslabs/aws-athena-query-federation - -# Parameters are CloudFormation features to pass input -# to your template when you create a stack -Parameters: - AthenaCatalogName: - Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' - Type: String - AllowedPattern: ^[a-z0-9-_]{1,64}$ - SpillBucket: - Description: 'The name of the bucket where this function can spill data.' - Type: String - VerticaExportBucket: - Description: "The bucket where the Vertica Query results will be exported." - Type: String - SpillPrefix: - Description: 'The prefix within SpillBucket where this function can spill data.' - Type: String - Default: athena-spill - LambdaTimeout: - Description: "Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)" - Default: 900 - Type: Number - LambdaMemory: - Description: "Lambda memory in MB (min 128 - 3008 max)." - Default: 3008 - Type: Number - DisableSpillEncryption: - Description: "WARNING: If set to 'true' encryption for spilled data is disabled." - Default: "false" - Type: String - VpcId: - Description: 'VPC ID' - Type: 'AWS::EC2::VPC::Id' - SubnetIds: - Description: 'One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' - Type: 'List' - SecretNameOrPrefix: - Description: 'The name or prefix of a set of names within Secrets Manager that this function should have access to. (e.g. vertica-*).' - Type: String - Default: "vertica-*" - VerticaConnectionString: - Description: 'The Vertica connection details to use by default if not catalog specific connection is defined and optionally using SecretsManager (e.g. ${secret_name}).' - Type: String - Default: "vertica://jdbc:vertica://:/?user=${vertica-username}&password=${vertica-password}" - PermissionsBoundaryARN: - Description: "(Optional) An IAM policy ARN to use as the PermissionsBoundary for the created Lambda function's execution role" - Default: '' - Type: String - -Conditions: - HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] - -Resources: - LambdaSecurityGroup: - Type: 'AWS::EC2::SecurityGroup' - Properties: - GroupDescription: 'Athena Vertica Connector Lambda VPC Security Group' - VpcId: !Ref VpcId - ConnectorConfig: - Type: 'AWS::Serverless::Function' - Properties: - Environment: - Variables: - disable_spill_encryption: !Ref DisableSpillEncryption - spill_bucket: !Ref SpillBucket - spill_prefix: !Ref SpillPrefix - export_bucket: !Ref VerticaExportBucket - default: !Ref VerticaConnectionString - - FunctionName: !Sub "${AthenaCatalogName}" - Handler: "com.amazonaws.athena.connectors.vertica.VerticaCompositeHandler" - CodeUri: "./target/athena-vertica-2022.47.1.jar" - Description: "Amazon Athena Vertica Connector" - Runtime: java11 - Timeout: !Ref LambdaTimeout - MemorySize: !Ref LambdaMemory - PermissionsBoundary: !If [ HasPermissionsBoundary, !Ref PermissionsBoundaryARN, !Ref "AWS::NoValue" ] - Policies: - - Statement: - - Action: - - athena:GetQueryExecution - Effect: Allow - Resource: '*' - Version: '2012-10-17' - - Statement: - - Action: - - s3:ListBucket - Effect: Allow - Resource: - - !Sub 'arn:${AWS::Partition}:s3:::${VerticaExportBucket}' - - !Sub 'arn:${AWS::Partition}:s3:::${SpillBucket}' - Version: '2012-10-17' - - Statement: - - Action: - - secretsmanager:GetSecretValue - Effect: Allow - Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:*:*:secret:${SecretNameOrPrefix}' - - S3ReadPolicy: - BucketName: - Ref: SpillBucket - - S3WritePolicy: - BucketName: - Ref: SpillBucket - - S3ReadPolicy: - BucketName: - Ref: VerticaExportBucket - - S3WritePolicy: - BucketName: - Ref: VerticaExportBucket - #VPCAccessPolicy allows our connector to run in a VPC so that it can access your data source. - VpcConfig: - SecurityGroupIds: - #SecurityGroup that should be applied to the Lambda function - - !Ref LambdaSecurityGroup - SubnetIds: !Ref SubnetIds \ No newline at end of file