-
Notifications
You must be signed in to change notification settings - Fork 300
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
162 changed files
with
7,466 additions
and
572 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,142 @@ | ||
| Transform: 'AWS::Serverless-2016-10-31' | ||
| Metadata: | ||
| 'AWS::ServerlessRepo::Application': | ||
| Name: AthenaAwsCmdbConnectorWithGlueConnection | ||
| Description: 'This connector enables Amazon Athena to communicate with various AWS Services, making your resource inventories accessible via SQL.' | ||
| Author: 'default author' | ||
| SpdxLicenseId: Apache-2.0 | ||
| LicenseUrl: LICENSE.txt | ||
| ReadmeUrl: README.md | ||
| Labels: | ||
| - athena-federation | ||
| HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' | ||
| SemanticVersion: 2022.47.1 | ||
| SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' | ||
| Parameters: | ||
| LambdaFunctionName: | ||
| Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' | ||
| Type: String | ||
| AllowedPattern: ^[a-z0-9-_]{1,64}$ | ||
| SpillBucket: | ||
| Description: 'The name of the bucket where this function can spill data.' | ||
| Type: String | ||
| GlueConnection: | ||
| Description: "Name of glue connection storing connection details for Federated Data source." | ||
| Type: String | ||
| KmsKeyId: | ||
| Description: "(Optional) By default any data that is spilled to S3 is encrypted using AES-GCM and a randomly generated key. Setting a KMS Key ID allows your Lambda function to use KMS for key generation for a stronger source of encryption keys." | ||
| Type: String | ||
| Default: "" | ||
| LambdaRoleArn: | ||
| Description: "(Optional) A custom role to be used by the Connector lambda" | ||
| Type: String | ||
| Default: "" | ||
|
|
||
| Conditions: | ||
| HasKmsKeyId: !Not [ !Equals [ !Ref KmsKeyId, "" ] ] | ||
| NotHasLambdaRole: !Equals [ !Ref LambdaRoleArn, "" ] | ||
| CreateKmsPolicy: !And [ !Condition HasKmsKeyId, !Condition NotHasLambdaRole ] | ||
| IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] | ||
| IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] | ||
|
|
||
| Resources: | ||
| ConnectorConfig: | ||
| Type: 'AWS::Serverless::Function' | ||
| Properties: | ||
| Environment: | ||
| Variables: | ||
| glue_connection: !Ref GlueConnection | ||
| FunctionName: !Ref LambdaFunctionName | ||
| PackageType: "Image" | ||
| ImageUri: !Sub | ||
| - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-aws-cmdb:2022.47.1' | ||
| - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] | ||
| Description: "Enables Amazon Athena to communicate with various AWS Services, making your resource inventories accessible via SQL." | ||
| Timeout: 900 | ||
| MemorySize: 3008 | ||
| Role: !If [NotHasLambdaRole, !GetAtt FunctionRole.Arn, !Ref LambdaRoleArn] | ||
|
|
||
| FunctionRole: | ||
| Condition: NotHasLambdaRole | ||
| Type: AWS::IAM::Role | ||
| Properties: | ||
| ManagedPolicyArns: | ||
| - "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" | ||
| AssumeRolePolicyDocument: | ||
| Version: 2012-10-17 | ||
| Statement: | ||
| - Effect: Allow | ||
| Principal: | ||
| Service: | ||
| - lambda.amazonaws.com | ||
| Action: | ||
| - "sts:AssumeRole" | ||
|
|
||
| FunctionExecutionPolicy: | ||
| Condition: NotHasLambdaRole | ||
| Type: "AWS::IAM::Policy" | ||
| Properties: | ||
| PolicyName: FunctionExecutionPolicy | ||
| PolicyDocument: | ||
| Version: 2012-10-17 | ||
| Statement: | ||
| - Action: | ||
| - autoscaling:Describe* | ||
| - elasticloadbalancing:Describe* | ||
| - ec2:Describe* | ||
| - elasticmapreduce:Describe* | ||
| - elasticmapreduce:List* | ||
| - rds:Describe* | ||
| - rds:ListTagsForResource | ||
| - athena:GetQueryExecution | ||
| - s3:ListBucket | ||
| - athena:GetQueryExecution | ||
| Effect: Allow | ||
| Resource: '*' | ||
| - Action: | ||
| - s3:GetObject | ||
| - s3:ListBucket | ||
| - s3:GetBucketLocation | ||
| - s3:GetObjectVersion | ||
| - s3:PutObject | ||
| - s3:PutObjectAcl | ||
| - s3:GetLifecycleConfiguration | ||
| - s3:PutLifecycleConfiguration | ||
| - s3:DeleteObject | ||
| Effect: Allow | ||
| Resource: | ||
| - Fn::Sub: | ||
| - arn:${AWS::Partition}:s3:::${bucketName} | ||
| - bucketName: | ||
| Ref: SpillBucket | ||
| - Fn::Sub: | ||
| - arn:${AWS::Partition}:s3:::${bucketName}/* | ||
| - bucketName: | ||
| Ref: SpillBucket | ||
| - Action: | ||
| - glue:GetConnection | ||
| Effect: Allow | ||
| Resource: | ||
| - !Sub 'arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:connection/${GlueConnection}' | ||
| - !Sub 'arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:catalog' | ||
| Roles: | ||
| - !Ref FunctionRole | ||
|
|
||
| FunctionKmsPolicy: | ||
| Condition: CreateKmsPolicy | ||
| Type: "AWS::IAM::Policy" | ||
| Properties: | ||
| PolicyName: FunctionKmsPolicy | ||
| PolicyDocument: | ||
| Version: 2012-10-17 | ||
| Statement: | ||
| - Effect: Allow | ||
| Action: | ||
| - kms:GenerateRandom | ||
| Resource: '*' | ||
| - Effect: Allow | ||
| Action: | ||
| - kms:GenerateDataKey | ||
| Resource: !Sub "arn:aws:kms:${AWS::Region}:${AWS::AccountId}:key/${KmsKeyId}" | ||
| Roles: | ||
| - !Ref FunctionRole |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
165 changes: 165 additions & 0 deletions
165
athena-cloudera-hive/athena-cloudera-hive-connection.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,165 @@ | ||
| Transform: 'AWS::Serverless-2016-10-31' | ||
| Metadata: | ||
| 'AWS::ServerlessRepo::Application': | ||
| Name: AthenaClouderaHiveConnectorWithGlueConnection | ||
| Description: 'This connector enables Amazon Athena to communicate with your Cloudera Hive instance(s) using JDBC driver.' | ||
| Author: 'default author' | ||
| SpdxLicenseId: Apache-2.0 | ||
| LicenseUrl: LICENSE.txt | ||
| ReadmeUrl: README.md | ||
| Labels: | ||
| - athena-federation | ||
| HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' | ||
| SemanticVersion: 2022.47.1 | ||
| SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' | ||
| Parameters: | ||
| LambdaFunctionName: | ||
| Description: 'This is the name of the lambda function that will be created. This name must satisfy the pattern ^[a-z0-9-_]{1,64}$' | ||
| Type: String | ||
| AllowedPattern: ^[a-z0-9-_]{1,64}$ | ||
| SecretName: | ||
| Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena JDBC Federation secret names can be prefixed with "AthenaJdbcFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaJdbcFederation*". Parameter value in this case should be "AthenaJdbcFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' | ||
| Type: String | ||
| SpillBucket: | ||
| Description: 'The name of the bucket where this function can spill data.' | ||
| Type: String | ||
| GlueConnection: | ||
| Description: "Name of glue connection storing connection details for Federated Data source." | ||
| Type: String | ||
| KmsKeyId: | ||
| Description: "(Optional) By default any data that is spilled to S3 is encrypted using AES-GCM and a randomly generated key. Setting a KMS Key ID allows your Lambda function to use KMS for key generation for a stronger source of encryption keys." | ||
| Type: String | ||
| Default: "" | ||
| LambdaRoleArn: | ||
| Description: "(Optional) A custom role to be used by the Connector lambda" | ||
| Type: String | ||
| Default: "" | ||
| SecurityGroupIds: | ||
| Description: 'One or more SecurityGroup IDs corresponding to the SecurityGroup that should be applied to the Lambda function. (e.g. sg1,sg2,sg3)' | ||
| Type: 'List<AWS::EC2::SecurityGroup::Id>' | ||
| SubnetIds: | ||
| Description: 'One or more Subnet IDs corresponding to the Subnet that the Lambda function can use to access you data source. (e.g. subnet1,subnet2)' | ||
| Type: 'List<AWS::EC2::Subnet::Id>' | ||
| Conditions: | ||
| NotHasLambdaRole: !Equals [ !Ref LambdaRoleArn, "" ] | ||
| HasKmsKeyId: !Not [ !Equals [ !Ref KmsKeyId, "" ] ] | ||
| CreateKmsPolicy: !And [!Condition NotHasLambdaRole, !Condition HasKmsKeyId] | ||
| IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] | ||
| IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] | ||
| Resources: | ||
| JdbcConnectorConfig: | ||
| Type: 'AWS::Serverless::Function' | ||
| Properties: | ||
| Environment: | ||
| Variables: | ||
| glue_connection: !Ref GlueConnection | ||
| FunctionName: !Ref LambdaFunctionName | ||
| PackageType: "Image" | ||
| ImageUri: !Sub | ||
| - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-cloudera-hive:2022.47.1' | ||
| - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] | ||
| ImageConfig: | ||
| Command: [ "com.amazonaws.athena.connectors.cloudera.HiveCompositeHandler" ] | ||
| Description: "Enables Amazon Athena to communicate with Cloudera Hive using JDBC" | ||
| Timeout: 900 | ||
| MemorySize: 3008 | ||
| Role: !If [NotHasLambdaRole, !GetAtt FunctionRole.Arn, !Ref LambdaRoleArn] | ||
| VpcConfig: | ||
| SecurityGroupIds: !Ref SecurityGroupIds | ||
| SubnetIds: !Ref SubnetIds | ||
|
|
||
| FunctionRole: | ||
| Condition: NotHasLambdaRole | ||
| Type: AWS::IAM::Role | ||
| Properties: | ||
| ManagedPolicyArns: | ||
| - "arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" | ||
| AssumeRolePolicyDocument: | ||
| Version: 2012-10-17 | ||
| Statement: | ||
| - Effect: Allow | ||
| Principal: | ||
| Service: | ||
| - lambda.amazonaws.com | ||
| Action: | ||
| - "sts:AssumeRole" | ||
|
|
||
| FunctionExecutionPolicy: | ||
| Condition: NotHasLambdaRole | ||
| Type: "AWS::IAM::Policy" | ||
| Properties: | ||
| Roles: | ||
| - !Ref FunctionRole | ||
| PolicyName: FunctionExecutionPolicy | ||
| PolicyDocument: | ||
| Version: 2012-10-17 | ||
| Statement: | ||
| - Action: | ||
| - secretsmanager:GetSecretValue | ||
| Effect: Allow | ||
| Resource: !Sub 'arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretName}*' | ||
| - Action: | ||
| - logs:CreateLogGroup | ||
| Effect: Allow | ||
| Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' | ||
| - Action: | ||
| - logs:CreateLogStream | ||
| - logs:PutLogEvents | ||
| Effect: Allow | ||
| Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*' | ||
| - Action: | ||
| - athena:GetQueryExecution | ||
| Effect: Allow | ||
| Resource: '*' | ||
| - Action: | ||
| - ec2:CreateNetworkInterface | ||
| - ec2:DeleteNetworkInterface | ||
| - ec2:DescribeNetworkInterfaces | ||
| - ec2:DetachNetworkInterface | ||
| Effect: Allow | ||
| Resource: '*' | ||
| - Action: | ||
| - s3:GetObject | ||
| - s3:ListBucket | ||
| - s3:GetBucketLocation | ||
| - s3:GetObjectVersion | ||
| - s3:PutObject | ||
| - s3:PutObjectAcl | ||
| - s3:GetLifecycleConfiguration | ||
| - s3:PutLifecycleConfiguration | ||
| - s3:DeleteObject | ||
| Effect: Allow | ||
| Resource: | ||
| - Fn::Sub: | ||
| - arn:${AWS::Partition}:s3:::${bucketName} | ||
| - bucketName: | ||
| Ref: SpillBucket | ||
| - Fn::Sub: | ||
| - arn:${AWS::Partition}:s3:::${bucketName}/* | ||
| - bucketName: | ||
| Ref: SpillBucket | ||
| - Action: | ||
| - glue:GetConnection | ||
| Effect: Allow | ||
| Resource: | ||
| - !Sub 'arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:connection/${GlueConnection}' | ||
| - !Sub 'arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:catalog' | ||
|
|
||
| FunctionKmsPolicy: | ||
| Condition: CreateKmsPolicy | ||
| Type: "AWS::IAM::Policy" | ||
| Properties: | ||
| PolicyName: FunctionKmsPolicy | ||
| PolicyDocument: | ||
| Version: 2012-10-17 | ||
| Statement: | ||
| - Effect: Allow | ||
| Action: | ||
| - kms:GenerateRandom | ||
| Resource: '*' | ||
| - Effect: Allow | ||
| Action: | ||
| - kms:GenerateDataKey | ||
| Resource: !Sub "arn:aws:kms:${AWS::Region}:${AWS::AccountId}:key/${KmsKeyId}" | ||
| Roles: | ||
| - !Ref FunctionRole |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.