diff --git a/client/client.go b/client/client.go index 231dbb078..35fac4a89 100644 --- a/client/client.go +++ b/client/client.go @@ -50,6 +50,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/route53domains" "github.com/aws/aws-sdk-go-v2/service/s3" s3control "github.com/aws/aws-sdk-go-v2/service/s3control" + "github.com/aws/aws-sdk-go-v2/service/sagemaker" "github.com/aws/aws-sdk-go-v2/service/sns" "github.com/aws/aws-sdk-go-v2/service/sqs" "github.com/aws/aws-sdk-go-v2/service/sts" @@ -126,6 +127,7 @@ type Services struct { S3 S3Client S3Control S3ControlClient S3Manager S3ManagerClient + SageMaker SageMakerClient SQS SQSClient Apigateway ApigatewayClient Apigatewayv2 Apigatewayv2Client @@ -136,6 +138,7 @@ type Services struct { Codebuild CodebuildClient GuardDuty GuardDutyClient } + type ServicesAccountRegionMap map[string]map[string]*Services // ServicesManager will hold the entire map of (account X region) services @@ -388,6 +391,7 @@ func initServices(region string, c aws.Config) Services { S3: s3.NewFromConfig(awsCfg), S3Control: s3control.NewFromConfig(awsCfg), S3Manager: newS3ManagerFromConfig(awsCfg), + SageMaker: sagemaker.NewFromConfig(awsCfg), SNS: sns.NewFromConfig(awsCfg), SQS: sqs.NewFromConfig(awsCfg), Waf: waf.NewFromConfig(awsCfg), diff --git a/client/mocks/mock_sagemaker.go b/client/mocks/mock_sagemaker.go new file mode 100644 index 000000000..5646c9c8a --- /dev/null +++ b/client/mocks/mock_sagemaker.go @@ -0,0 +1,216 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cloudquery/cq-provider-aws/client (interfaces: SageMakerClient) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + sagemaker "github.com/aws/aws-sdk-go-v2/service/sagemaker" + gomock "github.com/golang/mock/gomock" +) + +// MockSageMakerClient is a mock of SageMakerClient interface. +type MockSageMakerClient struct { + ctrl *gomock.Controller + recorder *MockSageMakerClientMockRecorder +} + +// MockSageMakerClientMockRecorder is the mock recorder for MockSageMakerClient. +type MockSageMakerClientMockRecorder struct { + mock *MockSageMakerClient +} + +// NewMockSageMakerClient creates a new mock instance. +func NewMockSageMakerClient(ctrl *gomock.Controller) *MockSageMakerClient { + mock := &MockSageMakerClient{ctrl: ctrl} + mock.recorder = &MockSageMakerClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSageMakerClient) EXPECT() *MockSageMakerClientMockRecorder { + return m.recorder +} + +// DescribeEndpointConfig mocks base method. +func (m *MockSageMakerClient) DescribeEndpointConfig(arg0 context.Context, arg1 *sagemaker.DescribeEndpointConfigInput, arg2 ...func(*sagemaker.Options)) (*sagemaker.DescribeEndpointConfigOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DescribeEndpointConfig", varargs...) + ret0, _ := ret[0].(*sagemaker.DescribeEndpointConfigOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DescribeEndpointConfig indicates an expected call of DescribeEndpointConfig. +func (mr *MockSageMakerClientMockRecorder) DescribeEndpointConfig(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeEndpointConfig", reflect.TypeOf((*MockSageMakerClient)(nil).DescribeEndpointConfig), varargs...) +} + +// DescribeModel mocks base method. +func (m *MockSageMakerClient) DescribeModel(arg0 context.Context, arg1 *sagemaker.DescribeModelInput, arg2 ...func(*sagemaker.Options)) (*sagemaker.DescribeModelOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DescribeModel", varargs...) + ret0, _ := ret[0].(*sagemaker.DescribeModelOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DescribeModel indicates an expected call of DescribeModel. +func (mr *MockSageMakerClientMockRecorder) DescribeModel(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeModel", reflect.TypeOf((*MockSageMakerClient)(nil).DescribeModel), varargs...) +} + +// DescribeNotebookInstance mocks base method. +func (m *MockSageMakerClient) DescribeNotebookInstance(arg0 context.Context, arg1 *sagemaker.DescribeNotebookInstanceInput, arg2 ...func(*sagemaker.Options)) (*sagemaker.DescribeNotebookInstanceOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DescribeNotebookInstance", varargs...) + ret0, _ := ret[0].(*sagemaker.DescribeNotebookInstanceOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DescribeNotebookInstance indicates an expected call of DescribeNotebookInstance. +func (mr *MockSageMakerClientMockRecorder) DescribeNotebookInstance(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeNotebookInstance", reflect.TypeOf((*MockSageMakerClient)(nil).DescribeNotebookInstance), varargs...) +} + +// DescribeTrainingJob mocks base method. +func (m *MockSageMakerClient) DescribeTrainingJob(arg0 context.Context, arg1 *sagemaker.DescribeTrainingJobInput, arg2 ...func(*sagemaker.Options)) (*sagemaker.DescribeTrainingJobOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DescribeTrainingJob", varargs...) + ret0, _ := ret[0].(*sagemaker.DescribeTrainingJobOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DescribeTrainingJob indicates an expected call of DescribeTrainingJob. +func (mr *MockSageMakerClientMockRecorder) DescribeTrainingJob(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeTrainingJob", reflect.TypeOf((*MockSageMakerClient)(nil).DescribeTrainingJob), varargs...) +} + +// ListEndpointConfigs mocks base method. +func (m *MockSageMakerClient) ListEndpointConfigs(arg0 context.Context, arg1 *sagemaker.ListEndpointConfigsInput, arg2 ...func(*sagemaker.Options)) (*sagemaker.ListEndpointConfigsOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListEndpointConfigs", varargs...) + ret0, _ := ret[0].(*sagemaker.ListEndpointConfigsOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListEndpointConfigs indicates an expected call of ListEndpointConfigs. +func (mr *MockSageMakerClientMockRecorder) ListEndpointConfigs(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListEndpointConfigs", reflect.TypeOf((*MockSageMakerClient)(nil).ListEndpointConfigs), varargs...) +} + +// ListModels mocks base method. +func (m *MockSageMakerClient) ListModels(arg0 context.Context, arg1 *sagemaker.ListModelsInput, arg2 ...func(*sagemaker.Options)) (*sagemaker.ListModelsOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListModels", varargs...) + ret0, _ := ret[0].(*sagemaker.ListModelsOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListModels indicates an expected call of ListModels. +func (mr *MockSageMakerClientMockRecorder) ListModels(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListModels", reflect.TypeOf((*MockSageMakerClient)(nil).ListModels), varargs...) +} + +// ListNotebookInstances mocks base method. +func (m *MockSageMakerClient) ListNotebookInstances(arg0 context.Context, arg1 *sagemaker.ListNotebookInstancesInput, arg2 ...func(*sagemaker.Options)) (*sagemaker.ListNotebookInstancesOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListNotebookInstances", varargs...) + ret0, _ := ret[0].(*sagemaker.ListNotebookInstancesOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListNotebookInstances indicates an expected call of ListNotebookInstances. +func (mr *MockSageMakerClientMockRecorder) ListNotebookInstances(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListNotebookInstances", reflect.TypeOf((*MockSageMakerClient)(nil).ListNotebookInstances), varargs...) +} + +// ListTags mocks base method. +func (m *MockSageMakerClient) ListTags(arg0 context.Context, arg1 *sagemaker.ListTagsInput, arg2 ...func(*sagemaker.Options)) (*sagemaker.ListTagsOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListTags", varargs...) + ret0, _ := ret[0].(*sagemaker.ListTagsOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListTags indicates an expected call of ListTags. +func (mr *MockSageMakerClientMockRecorder) ListTags(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListTags", reflect.TypeOf((*MockSageMakerClient)(nil).ListTags), varargs...) +} + +// ListTrainingJobs mocks base method. +func (m *MockSageMakerClient) ListTrainingJobs(arg0 context.Context, arg1 *sagemaker.ListTrainingJobsInput, arg2 ...func(*sagemaker.Options)) (*sagemaker.ListTrainingJobsOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListTrainingJobs", varargs...) + ret0, _ := ret[0].(*sagemaker.ListTrainingJobsOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListTrainingJobs indicates an expected call of ListTrainingJobs. +func (mr *MockSageMakerClientMockRecorder) ListTrainingJobs(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListTrainingJobs", reflect.TypeOf((*MockSageMakerClient)(nil).ListTrainingJobs), varargs...) +} diff --git a/client/services.go b/client/services.go index 8335f61c8..e4d68432a 100644 --- a/client/services.go +++ b/client/services.go @@ -40,6 +40,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/route53domains" "github.com/aws/aws-sdk-go-v2/service/s3" s3control "github.com/aws/aws-sdk-go-v2/service/s3control" + "github.com/aws/aws-sdk-go-v2/service/sagemaker" "github.com/aws/aws-sdk-go-v2/service/sns" "github.com/aws/aws-sdk-go-v2/service/sqs" "github.com/aws/aws-sdk-go-v2/service/waf" @@ -458,6 +459,19 @@ type GuardDutyClient interface { GetDetector(ctx context.Context, params *guardduty.GetDetectorInput, optFns ...func(*guardduty.Options)) (*guardduty.GetDetectorOutput, error) } +//go:generate mockgen -package=mocks -destination=./mocks/mock_sagemaker.go . SageMakerClient +type SageMakerClient interface { + ListNotebookInstances(ctx context.Context, params *sagemaker.ListNotebookInstancesInput, optFns ...func(*sagemaker.Options)) (*sagemaker.ListNotebookInstancesOutput, error) + DescribeNotebookInstance(ctx context.Context, params *sagemaker.DescribeNotebookInstanceInput, optFns ...func(*sagemaker.Options)) (*sagemaker.DescribeNotebookInstanceOutput, error) + ListTags(ctx context.Context, params *sagemaker.ListTagsInput, optFns ...func(*sagemaker.Options)) (*sagemaker.ListTagsOutput, error) + ListModels(ctx context.Context, params *sagemaker.ListModelsInput, optFns ...func(*sagemaker.Options)) (*sagemaker.ListModelsOutput, error) + DescribeModel(ctx context.Context, params *sagemaker.DescribeModelInput, optFns ...func(*sagemaker.Options)) (*sagemaker.DescribeModelOutput, error) + ListEndpointConfigs(ctx context.Context, params *sagemaker.ListEndpointConfigsInput, optFns ...func(*sagemaker.Options)) (*sagemaker.ListEndpointConfigsOutput, error) + DescribeEndpointConfig(ctx context.Context, params *sagemaker.DescribeEndpointConfigInput, optFns ...func(*sagemaker.Options)) (*sagemaker.DescribeEndpointConfigOutput, error) + ListTrainingJobs(ctx context.Context, params *sagemaker.ListTrainingJobsInput, optFns ...func(*sagemaker.Options)) (*sagemaker.ListTrainingJobsOutput, error) + DescribeTrainingJob(ctx context.Context, params *sagemaker.DescribeTrainingJobInput, optFns ...func(*sagemaker.Options)) (*sagemaker.DescribeTrainingJobOutput, error) +} + //go:generate mockgen -package=mocks -destination=./mocks/codebuild.go . CodebuildClient type CodebuildClient interface { BatchGetProjects(ctx context.Context, params *codebuild.BatchGetProjectsInput, optFns ...func(*codebuild.Options)) (*codebuild.BatchGetProjectsOutput, error) diff --git a/docs/tables/aws_sagemaker_endpoint_configuration_production_variants.md b/docs/tables/aws_sagemaker_endpoint_configuration_production_variants.md new file mode 100644 index 000000000..782b65ea0 --- /dev/null +++ b/docs/tables/aws_sagemaker_endpoint_configuration_production_variants.md @@ -0,0 +1,15 @@ + +# Table: aws_sagemaker_endpoint_configuration_production_variants +Identifies a model that you want to host and the resources chosen to deploy for hosting it +## Columns +| Name | Type | Description | +| ------------- | ------------- | ----- | +|endpoint_configuration_cq_id|uuid|Unique CloudQuery ID of aws_sagemaker_endpoint_configurations table (FK)| +|initial_instance_count|integer|Number of instances to launch initially. This member is required.| +|instance_type|text|The ML compute instance type. This member is required.| +|model_name|text|The name of the model that you want to host| +|variant_name|text|The name of the production variant. This member is required.| +|accelerator_type|text|The size of the Elastic Inference (EI) instance to use for the production variant| +|core_dump_config_destination_s3_uri|text|The Amazon S3 bucket to send the core dump to. This member is required.| +|core_dump_config_kms_key_id|text|The Amazon Web Services Key Management Service (Amazon Web Services KMS) key that Amazon SageMaker uses to encrypt the core dump data at rest using Amazon S3 server-side encryption| +|initial_variant_weight|float|Determines initial traffic distribution among all of the models that you specify in the endpoint configuration| diff --git a/docs/tables/aws_sagemaker_endpoint_configurations.md b/docs/tables/aws_sagemaker_endpoint_configurations.md new file mode 100644 index 000000000..1ddded237 --- /dev/null +++ b/docs/tables/aws_sagemaker_endpoint_configurations.md @@ -0,0 +1,14 @@ + +# Table: aws_sagemaker_endpoint_configurations +Provides summary information for an endpoint configuration. +## Columns +| Name | Type | Description | +| ------------- | ------------- | ----- | +|account_id|text|The AWS Account ID of the resource.| +|region|text|The AWS Region of the resource.| +|kms_key_id|text|Amazon Web Services KMS key ID Amazon SageMaker uses to encrypt data when storing it on the ML storage volume attached to the instance.| +|data_capture_config|jsonb|| +|tags|jsonb|The tags associated with the model.| +|creation_time|timestamp without time zone|A timestamp that indicates when the endpoint configuration was created.| +|arn|text|The Amazon Resource Name (ARN) of the endpoint configuration.| +|name|text|Name of the Amazon SageMaker endpoint configuration.| diff --git a/docs/tables/aws_sagemaker_model_containers.md b/docs/tables/aws_sagemaker_model_containers.md new file mode 100644 index 000000000..b473cae53 --- /dev/null +++ b/docs/tables/aws_sagemaker_model_containers.md @@ -0,0 +1,16 @@ + +# Table: aws_sagemaker_model_containers +Describes the container, as part of model definition. +## Columns +| Name | Type | Description | +| ------------- | ------------- | ----- | +|model_cq_id|uuid|Unique CloudQuery ID of aws_sagemaker_model table (FK)| +|container_hostname|text|This parameter is ignored for models that contain only a PrimaryContainer| +|environment|jsonb|The environment variables to set in the Docker container| +|image|text|The path where inference code is stored| +|image_config_repository_access_mode|text|Set this to one of the following values: * Platform - The model image is hosted in Amazon ECR. * Vpc - The model image is hosted in a private Docker registry in your VPC. This member is required.| +|image_config_repository_auth_config_repository_credentials_provider_arn|text|The Amazon Resource Name (ARN) of an Amazon Web Services Lambda function that provides credentials to authenticate to the private Docker registry where your model image is hosted| +|mode|text|Whether the container hosts a single model or multiple models.| +|model_data_url|text|The S3 path where the model artifacts, which result from model training, are stored| +|model_package_name|text|The name or Amazon Resource Name (ARN) of the model package to use to create the model.| +|multi_model_config_model_cache_setting|text|Whether to cache models for a multi-model endpoint| diff --git a/docs/tables/aws_sagemaker_model_vpc_config.md b/docs/tables/aws_sagemaker_model_vpc_config.md new file mode 100644 index 000000000..dd0c2098a --- /dev/null +++ b/docs/tables/aws_sagemaker_model_vpc_config.md @@ -0,0 +1,9 @@ + +# Table: aws_sagemaker_model_vpc_config +Specifies a VPC that your training jobs and hosted models have access to. Control access to and from your training and model containers by configuring the VPC +## Columns +| Name | Type | Description | +| ------------- | ------------- | ----- | +|model_cq_id|uuid|Unique CloudQuery ID of aws_sagemaker_model table (FK)| +|security_group_ids|text[]|The VPC security group IDs, in the form sg-xxxxxxxx| +|subnets|text[]|The ID of the subnets in the VPC to which you want to connect your training job or model| diff --git a/docs/tables/aws_sagemaker_models.md b/docs/tables/aws_sagemaker_models.md new file mode 100644 index 000000000..ed8a3cb49 --- /dev/null +++ b/docs/tables/aws_sagemaker_models.md @@ -0,0 +1,16 @@ + +# Table: aws_sagemaker_models +Provides summary information about a model. +## Columns +| Name | Type | Description | +| ------------- | ------------- | ----- | +|account_id|text|The AWS Account ID of the resource.| +|region|text|The AWS Region of the resource.| +|enable_network_isolation|boolean|If True, no inbound or outbound network calls can be made to or from the model container.| +|execution_role_arn|text|The Amazon Resource Name (ARN) of the IAM role that you specified for the model.| +|inference_execution_config|jsonb|Specifies details of how containers in a multi-container endpoint are called.| +|primary_container|jsonb|The location of the primary inference code, associated artifacts, and custom environment map that the inference code uses when it is deployed in production.| +|tags|jsonb|The tags associated with the model.| +|creation_time|timestamp without time zone|A timestamp that indicates when the model was created.| +|arn|text|The Amazon Resource Name (ARN) of the model.| +|name|text|The name of the model that you want a summary for.| diff --git a/docs/tables/aws_sagemaker_notebook_instances.md b/docs/tables/aws_sagemaker_notebook_instances.md new file mode 100644 index 000000000..eb723cd2b --- /dev/null +++ b/docs/tables/aws_sagemaker_notebook_instances.md @@ -0,0 +1,26 @@ + +# Table: aws_sagemaker_notebook_instances +Provides summary information for an Amazon SageMaker notebook instance. +## Columns +| Name | Type | Description | +| ------------- | ------------- | ----- | +|account_id|text|The AWS Account ID of the resource.| +|region|text|The AWS Region of the resource.| +|network_interface_id|text|The network interface IDs that Amazon SageMaker created at the time of creating the instance.| +|kms_key_id|text|The Amazon Web Services KMS key ID Amazon SageMaker uses to encrypt data when storing it on the ML storage volume attached to the instance.| +|subnet_id|text|The ID of the VPC subnet.| +|volume_size_in_gb|integer|The size, in GB, of the ML storage volume attached to the notebook instance.| +|accelerator_types|text[]|A list of the Elastic Inference (EI) instance types associated with this notebook instance.| +|security_groups|jsonb|The IDs of the VPC security groups.| +|direct_internet_access|boolean|Describes whether Amazon SageMaker provides internet access to the notebook instance.| +|tags|jsonb|The tags associated with the notebook instance.| +|arn|text|The Amazon Resource Name (ARN) of the notebook instance. This member is required.| +|name|text|The name of the notebook instance that you want a summary for. This member is required.| +|additional_code_repositories|text[]|An array of up to three Git repositories associated with the notebook instance. These can be either the names of Git repositories stored as resources in your account, or the URL of Git repositories in Amazon Web Services CodeCommit (https://docs.aws.amazon.com/codecommit/latest/userguide/welcome.html) or in any other Git repository| +|creation_time|timestamp without time zone|A timestamp that shows when the notebook instance was created.| +|default_code_repository|text|The Git repository associated with the notebook instance as its default code repository| +|instance_type|text|The type of ML compute instance that the notebook instance is running on.| +|last_modified_time|timestamp without time zone|A timestamp that shows when the notebook instance was last modified.| +|notebook_instance_lifecycle_config_name|text|The name of a notebook instance lifecycle configuration associated with this notebook instance| +|notebook_instance_status|text|The status of the notebook instance.| +|url|text|The URL that you use to connect to the Jupyter instance running in your notebook instance.| diff --git a/docs/tables/aws_sagemaker_training_job_algorithm_specification.md b/docs/tables/aws_sagemaker_training_job_algorithm_specification.md new file mode 100644 index 000000000..50e8eaeff --- /dev/null +++ b/docs/tables/aws_sagemaker_training_job_algorithm_specification.md @@ -0,0 +1,12 @@ + +# Table: aws_sagemaker_training_job_algorithm_specification +Specifies the training algorithm to use in a CreateTrainingJob request +## Columns +| Name | Type | Description | +| ------------- | ------------- | ----- | +|training_job_cq_id|uuid|Unique CloudQuery ID of aws_sagemaker_training_jobs table (FK)| +|training_input_mode|text|The training input mode that the algorithm supports| +|algorithm_name|text|The name of the algorithm resource to use for the training job| +|enable_sage_maker_metrics_time_series|boolean|To generate and save time-series metrics during training, set to true| +|metric_definitions|jsonb|A list of metric definition objects| +|training_image|text|The registry path of the Docker image that contains the training algorithm| diff --git a/docs/tables/aws_sagemaker_training_job_debug_hook_config.md b/docs/tables/aws_sagemaker_training_job_debug_hook_config.md new file mode 100644 index 000000000..b0b64f5b9 --- /dev/null +++ b/docs/tables/aws_sagemaker_training_job_debug_hook_config.md @@ -0,0 +1,11 @@ + +# Table: aws_sagemaker_training_job_debug_hook_config +Configuration information for the Debugger hook parameters, metric and tensor collections, and storage paths +## Columns +| Name | Type | Description | +| ------------- | ------------- | ----- | +|training_job_cq_id|uuid|Unique CloudQuery ID of aws_sagemaker_training_jobs table (FK)| +|s3_output_path|text|Path to Amazon S3 storage location for metrics and tensors. This member is required.| +|collection_configurations|jsonb|Configuration information for Debugger tensor collections| +|hook_parameters|jsonb|Configuration information for the Debugger hook parameters.| +|local_path|text|Path to local storage location for metrics and tensors| diff --git a/docs/tables/aws_sagemaker_training_job_debug_rule_configurations.md b/docs/tables/aws_sagemaker_training_job_debug_rule_configurations.md new file mode 100644 index 000000000..1fac4ae25 --- /dev/null +++ b/docs/tables/aws_sagemaker_training_job_debug_rule_configurations.md @@ -0,0 +1,14 @@ + +# Table: aws_sagemaker_training_job_debug_rule_configurations +Configuration information for SageMaker Debugger rules for debugging +## Columns +| Name | Type | Description | +| ------------- | ------------- | ----- | +|training_job_cq_id|uuid|Unique CloudQuery ID of aws_sagemaker_training_jobs table (FK)| +|rule_configuration_name|text|The name of the rule configuration| +|rule_evaluator_image|text|The Amazon Elastic Container (ECR) Image for the managed rule evaluation. This member is required.| +|instance_type|text|The instance type to deploy a Debugger custom rule for debugging a training job.| +|local_path|text|Path to local storage location for output of rules| +|rule_parameters|jsonb|Runtime configuration for rule container.| +|s3_output_path|text|Path to Amazon S3 storage location for rules.| +|volume_size_in_gb|integer|The size, in GB, of the ML storage volume attached to the processing instance.| diff --git a/docs/tables/aws_sagemaker_training_job_debug_rule_evaluation_statuses.md b/docs/tables/aws_sagemaker_training_job_debug_rule_evaluation_statuses.md new file mode 100644 index 000000000..ab8f3c701 --- /dev/null +++ b/docs/tables/aws_sagemaker_training_job_debug_rule_evaluation_statuses.md @@ -0,0 +1,12 @@ + +# Table: aws_sagemaker_training_job_debug_rule_evaluation_statuses +Information about the status of the rule evaluation. +## Columns +| Name | Type | Description | +| ------------- | ------------- | ----- | +|training_job_cq_id|uuid|Unique CloudQuery ID of aws_sagemaker_training_jobs table (FK)| +|last_modified_time|timestamp without time zone|Timestamp when the rule evaluation status was last modified.| +|rule_configuration_name|text|The name of the rule configuration.| +|rule_evaluation_job_arn|text|The Amazon Resource Name (ARN) of the rule evaluation job.| +|rule_evaluation_status|text|Status of the rule evaluation.| +|status_details|text|Details from the rule evaluation.| diff --git a/docs/tables/aws_sagemaker_training_job_input_data_config.md b/docs/tables/aws_sagemaker_training_job_input_data_config.md new file mode 100644 index 000000000..71ce0c055 --- /dev/null +++ b/docs/tables/aws_sagemaker_training_job_input_data_config.md @@ -0,0 +1,21 @@ + +# Table: aws_sagemaker_training_job_input_data_config +A channel is a named input source that training algorithms can consume. +## Columns +| Name | Type | Description | +| ------------- | ------------- | ----- | +|training_job_cq_id|uuid|Unique CloudQuery ID of aws_sagemaker_training_jobs table (FK)| +|channel_name|text|The name of the channel. This member is required.| +|data_source_file_directory_path|text|The full path to the directory to associate with the channel. This member is required.| +|data_source_file_system_access_mode|text|The access mode of the mount of the directory associated with the channel| +|data_source_file_system_id|text|The file system id. This member is required.| +|data_source_file_system_type|text|The file system type. This member is required.| +|data_source_s3_data_type|text|If you choose S3Prefix, S3Uri identifies a key name prefix| +|data_source_s3_uri|text|Depending on the value specified for the S3DataType, identifies either a key name prefix or a manifest| +|data_source_attribute_names|text[]|A list of one or more attribute names to use that are found in a specified augmented manifest file.| +|data_source_s3_data_distribution_type|text|If you want Amazon SageMaker to replicate the entire dataset on each ML compute instance that is launched for model training, specify FullyReplicated| +|compression_type|text|If training data is compressed, the compression type| +|content_type|text|The MIME type of the data.| +|input_mode|text|(Optional) The input mode to use for the data channel in a training job| +|record_wrapper_type|text|Specify RecordIO as the value when input data is in raw format but the training algorithm requires the RecordIO format| +|shuffle_config_seed|bigint|Determines the shuffling order in ShuffleConfig value. This member is required.| diff --git a/docs/tables/aws_sagemaker_training_job_profiler_rule_configurations.md b/docs/tables/aws_sagemaker_training_job_profiler_rule_configurations.md new file mode 100644 index 000000000..68adb4505 --- /dev/null +++ b/docs/tables/aws_sagemaker_training_job_profiler_rule_configurations.md @@ -0,0 +1,14 @@ + +# Table: aws_sagemaker_training_job_profiler_rule_configurations +Configuration information for profiling rules. +## Columns +| Name | Type | Description | +| ------------- | ------------- | ----- | +|training_job_cq_id|uuid|Unique CloudQuery ID of aws_sagemaker_training_jobs table (FK)| +|rule_configuration_name|text|The name of the rule configuration| +|rule_evaluator_image|text|The Amazon Elastic Container (ECR) Image for the managed rule evaluation. This member is required.| +|instance_type|text|The instance type to deploy a Debugger custom rule for profiling a training job.| +|local_path|text|Path to local storage location for output of rules| +|rule_parameters|jsonb|Runtime configuration for rule container.| +|s3_output_path|text|Path to Amazon S3 storage location for rules.| +|volume_size_in_gb|integer|The size, in GB, of the ML storage volume attached to the processing instance.| diff --git a/docs/tables/aws_sagemaker_training_job_profiler_rule_evaluation_statuses.md b/docs/tables/aws_sagemaker_training_job_profiler_rule_evaluation_statuses.md new file mode 100644 index 000000000..a06e2fd69 --- /dev/null +++ b/docs/tables/aws_sagemaker_training_job_profiler_rule_evaluation_statuses.md @@ -0,0 +1,12 @@ + +# Table: aws_sagemaker_training_job_profiler_rule_evaluation_statuses +Information about the status of the rule evaluation. +## Columns +| Name | Type | Description | +| ------------- | ------------- | ----- | +|training_job_cq_id|uuid|Unique CloudQuery ID of aws_sagemaker_training_jobs table (FK)| +|last_modified_time|timestamp without time zone|Timestamp when the rule evaluation status was last modified.| +|rule_configuration_name|text|The name of the rule configuration.| +|rule_evaluation_job_arn|text|The Amazon Resource Name (ARN) of the rule evaluation job.| +|rule_evaluation_status|text|Status of the rule evaluation.| +|status_details|text|Details from the rule evaluation.| diff --git a/docs/tables/aws_sagemaker_training_jobs.md b/docs/tables/aws_sagemaker_training_jobs.md new file mode 100644 index 000000000..2a49140d8 --- /dev/null +++ b/docs/tables/aws_sagemaker_training_jobs.md @@ -0,0 +1,41 @@ + +# Table: aws_sagemaker_training_jobs +Provides summary information about a training job. +## Columns +| Name | Type | Description | +| ------------- | ------------- | ----- | +|account_id|text|The AWS Account ID of the resource.| +|region|text|The AWS Region of the resource.| +|auto_ml_job_arn|text|The Amazon Resource Name (ARN) of an AutoML job.| +|billable_time_in_seconds|integer|The billable time in seconds. Billable time refers to the absolute wall-clock time.| +|enable_managed_spot_training|boolean|A Boolean indicating whether managed spot training is enabled (True) or not (False).| +|enable_network_isolation|boolean|If you want to allow inbound or outbound network calls, except for calls between peers within a training cluster for distributed training, choose True. If you enable network isolation for training jobs that are configured to use a VPC, Amazon SageMaker downloads and uploads customer data and model artifacts through the specified VPC, but the training container does not have network access.| +|enable_inter_container_traffic_encryption|boolean|To encrypt all communications between ML compute instances in distributed training, choose True. Encryption provides greater security for distributed training, but training might take longer. How long it takes depends on the amount of communication between compute instances, especially if you use a deep learning algorithms in distributed training.| +|failure_reason|text|If the training job failed, the reason it failed.| +|labeling_job_arn|text|The Amazon Resource Name (ARN) of the Amazon SageMaker Ground Truth labeling job that created the transform or training job.| +|last_modified_time|timestamp without time zone|A timestamp that indicates when the status of the training job was last modified.| +|profiling_status|text|Profiling status of a training job.| +|role_arn|text|The Amazon Web Services Identity and Access Management (IAM) role configured for the training job.| +|secondary_status|text|Provides detailed information about the state of the training job.| +|training_end_time|timestamp without time zone|Indicates the time when the training job ends on training instances.| +|training_start_time|timestamp without time zone|Indicates the time when the training job starts on training instances.| +|training_time_in_seconds|integer|The training time in seconds.| +|tuning_job_arn|text|The Amazon Resource Name (ARN) of the associated hyperparameter tuning job if the training job was launched by a hyperparameter tuning job.| +|checkpoint_config|jsonb|Contains information about the output location for managed spot training checkpoint data.| +|environment|jsonb|The environment variables to set in the Docker container.| +|experiment_config|jsonb|Associates a SageMaker job as a trial component with an experiment and trial.| +|hyper_parameters|jsonb|Algorithm-specific parameters.| +|model_artifacts|jsonb|Information about the Amazon S3 location that is configured for storing model artifacts.| +|output_data_config|jsonb|The S3 path where model artifacts that you configured when creating the job are stored.| +|profiler_config|jsonb|Configuration information for Debugger system monitoring, framework profiling, and storage paths.| +|resource_config|jsonb|Resources, including ML compute instances and ML storage volumes, that are configured for model training.| +|stopping_condition|jsonb|Specifies a limit to how long a model training job can run.| +|tensor_board_output_config|jsonb|Configuration of storage locations for the Debugger TensorBoard output data.| +|vpc_config|jsonb|A VpcConfig object that specifies the VPC that this training job has access to.| +|tags|jsonb|The tags associated with the model.| +|creation_time|timestamp without time zone|A timestamp that shows when the training job was created.| +|arn|text|The Amazon Resource Name (ARN) of the training job.| +|name|text|The name of the training job.| +|training_job_status|text|The status of the training job.| +|secondary_status_transitions|jsonb|| +|final_metric_data_list|jsonb|| diff --git a/go.mod b/go.mod index f24a6741c..8d0bb58b5 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/redshift v1.3.0 github.com/aws/aws-sdk-go-v2/service/route53 v1.4.0 github.com/aws/aws-sdk-go-v2/service/s3 v1.19.1 + github.com/aws/aws-sdk-go-v2/service/sagemaker v1.19.1 github.com/aws/aws-sdk-go-v2/service/s3control v1.14.1 github.com/aws/aws-sdk-go-v2/service/sns v1.1.2 github.com/aws/aws-sdk-go-v2/service/sqs v1.9.1 diff --git a/go.sum b/go.sum index 80c149f08..0dddeafcb 100644 --- a/go.sum +++ b/go.sum @@ -251,6 +251,8 @@ github.com/aws/aws-sdk-go-v2/service/s3 v1.19.1 h1:v7n7a2v9fN+We4Jna/u7+35Fhch5Y github.com/aws/aws-sdk-go-v2/service/s3 v1.19.1/go.mod h1:wcAYHjbvrLxDNWJmwCgwxudlHIkSLyU2m4Q1tWO6QZw= github.com/aws/aws-sdk-go-v2/service/s3control v1.14.1 h1:Nmcb6pxJtjJof+mmF9TJvyWuSbzv7sCn5YoK3MAsPek= github.com/aws/aws-sdk-go-v2/service/s3control v1.14.1/go.mod h1:dTnxIRqR69JUZobQDUh47rlbYe8PzTd0k4o+gDkHeV4= +github.com/aws/aws-sdk-go-v2/service/sagemaker v1.19.1 h1:cy6fUlP94vzD/0VUD3SWGUBfYrOr+zP+ChsTxUtZydQ= +github.com/aws/aws-sdk-go-v2/service/sagemaker v1.19.1/go.mod h1:G9AcXDbGtZVA8XBdmpbVQv1lvmiuk4I9n2MQlp1FJ9k= github.com/aws/aws-sdk-go-v2/service/sns v1.1.2 h1:1U/FujyBEkNwrvANUcZFuVnAQqy0EAUEGToso5Dcijs= github.com/aws/aws-sdk-go-v2/service/sns v1.1.2/go.mod h1:/vvAGyo3/TG5CSrJQarIlwzjE6O/DjBIvJTRkpYkvwA= github.com/aws/aws-sdk-go-v2/service/sqs v1.9.1 h1:8m+6iuSldxMrVQbjHRcWPnUxdpD3RCPtacmFFNkR4Vw= diff --git a/resources/integration_tests/aws_sagemaker_endpoint_configurations_test.go b/resources/integration_tests/aws_sagemaker_endpoint_configurations_test.go new file mode 100644 index 000000000..401821ef9 --- /dev/null +++ b/resources/integration_tests/aws_sagemaker_endpoint_configurations_test.go @@ -0,0 +1,47 @@ +package integration_tests + +import ( + "fmt" + "testing" + + "github.com/cloudquery/cq-provider-aws/resources" + providertest "github.com/cloudquery/cq-provider-sdk/provider/testing" +) + +func TestIntegrationSageMakerEndpointConfigurations(t *testing.T) { + awsTestIntegrationHelper(t, resources.SagemakerEndpointConfigurations(), []string{"aws_sagemaker_endpoint_configurations.tf", "aws_sagemaker_models.tf"}, func(res *providertest.ResourceIntegrationTestData) providertest.ResourceIntegrationVerification { + return providertest.ResourceIntegrationVerification{ + Name: "aws_sagemaker_endpoint_configurations", + ExpectedValues: []providertest.ExpectedValue{ + { + Count: 1, + Data: map[string]interface{}{ + "name": fmt.Sprintf("sagemaker-endpoint-configuration-%s%s", res.Prefix, res.Suffix), + "tags": map[string]interface{}{ + "TestId": res.Suffix, + "Type": "integration_test", + "Name": fmt.Sprintf("sagemaker-endpoint-configuration-%s%s", res.Prefix, res.Suffix), + }, + }, + }, + }, + Relations: []*providertest.ResourceIntegrationVerification{ + { + Name: "aws_sagemaker_endpoint_configuration_production_variants", + ForeignKeyName: "endpoint_configuration_cq_id", + ExpectedValues: []providertest.ExpectedValue{ + { + Count: 1, + Data: map[string]interface{}{ + "variant_name": fmt.Sprintf("variant-%s%s", res.Prefix, res.Suffix), + "model_name": fmt.Sprintf("sagemaker-model-%s%s", res.Prefix, res.Suffix), + "initial_instance_count": float64(1), + "instance_type": "ml.t2.medium", + }, + }, + }, + }, + }, + } + }) +} diff --git a/resources/integration_tests/aws_sagemaker_models_test.go b/resources/integration_tests/aws_sagemaker_models_test.go new file mode 100644 index 000000000..e4df67bb1 --- /dev/null +++ b/resources/integration_tests/aws_sagemaker_models_test.go @@ -0,0 +1,30 @@ +package integration_tests + +import ( + "fmt" + "testing" + + "github.com/cloudquery/cq-provider-aws/resources" + providertest "github.com/cloudquery/cq-provider-sdk/provider/testing" +) + +func TestIntegrationSageMakerModels(t *testing.T) { + awsTestIntegrationHelper(t, resources.SagemakerModels(), nil, func(res *providertest.ResourceIntegrationTestData) providertest.ResourceIntegrationVerification { + return providertest.ResourceIntegrationVerification{ + Name: "aws_sagemaker_models", + ExpectedValues: []providertest.ExpectedValue{ + { + Count: 1, + Data: map[string]interface{}{ + "name": fmt.Sprintf("sagemaker-model-%s%s", res.Prefix, res.Suffix), + "tags": map[string]interface{}{ + "TestId": res.Suffix, + "Type": "integration_test", + "Name": fmt.Sprintf("sagemaker-model-%s%s", res.Prefix, res.Suffix), + }, + }, + }, + }, + } + }) +} diff --git a/resources/integration_tests/aws_sagemaker_notebook_instances_test.go b/resources/integration_tests/aws_sagemaker_notebook_instances_test.go new file mode 100644 index 000000000..d9a6c7325 --- /dev/null +++ b/resources/integration_tests/aws_sagemaker_notebook_instances_test.go @@ -0,0 +1,31 @@ +package integration_tests + +import ( + "fmt" + "testing" + + "github.com/cloudquery/cq-provider-aws/resources" + providertest "github.com/cloudquery/cq-provider-sdk/provider/testing" +) + +func TestIntegrationSageMakerNotebookInstances(t *testing.T) { + awsTestIntegrationHelper(t, resources.SagemakerNotebookInstances(), nil, func(res *providertest.ResourceIntegrationTestData) providertest.ResourceIntegrationVerification { + return providertest.ResourceIntegrationVerification{ + Name: "aws_sagemaker_notebook_instances", + ExpectedValues: []providertest.ExpectedValue{ + { + Count: 1, + Data: map[string]interface{}{ + "name": fmt.Sprintf("sagemaker-%s%s", res.Prefix, res.Suffix), + "instance_type": "ml.t2.medium", + "tags": map[string]interface{}{ + "TestId": res.Suffix, + "Type": "integration_test", + "Name": fmt.Sprintf("sagemaker-%s%s", res.Prefix, res.Suffix), + }, + }, + }, + }, + } + }) +} diff --git a/resources/integration_tests/infra/aws_sagemaker_endpoint_configurations.tf b/resources/integration_tests/infra/aws_sagemaker_endpoint_configurations.tf new file mode 100644 index 000000000..b62377467 --- /dev/null +++ b/resources/integration_tests/infra/aws_sagemaker_endpoint_configurations.tf @@ -0,0 +1,14 @@ +resource "aws_sagemaker_endpoint_configuration" "sagemaker_endpoint_configuration" { + name = "sagemaker-endpoint-configuration-${var.test_prefix}${var.test_suffix}" + + production_variants { + variant_name = "variant-${var.test_prefix}${var.test_suffix}" + model_name = aws_sagemaker_model.sagemaker_model.name + initial_instance_count = 1 + instance_type = "ml.t2.medium" + } + + tags = { + Name = "sagemaker-endpoint-configuration-${var.test_prefix}${var.test_suffix}" + } +} \ No newline at end of file diff --git a/resources/integration_tests/infra/aws_sagemaker_models.tf b/resources/integration_tests/infra/aws_sagemaker_models.tf new file mode 100644 index 000000000..24386e8cb --- /dev/null +++ b/resources/integration_tests/infra/aws_sagemaker_models.tf @@ -0,0 +1,31 @@ +resource "aws_sagemaker_model" "sagemaker_model" { + name = "sagemaker-model-${var.test_prefix}${var.test_suffix}" + execution_role_arn = aws_iam_role.sagemaker_model_iam_role.arn + + primary_container { + image = data.aws_sagemaker_prebuilt_ecr_image.sagemaker_model_ecr_image.registry_path + } + + tags = { + Name = "sagemaker-model-${var.test_prefix}${var.test_suffix}" + } +} + +resource "aws_iam_role" "sagemaker_model_iam_role" { + assume_role_policy = data.aws_iam_policy_document.sagemaker_model_assume_role.json +} + +data "aws_iam_policy_document" "sagemaker_model_assume_role" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["sagemaker.amazonaws.com"] + } + } +} + +data "aws_sagemaker_prebuilt_ecr_image" "sagemaker_model_ecr_image" { + repository_name = "kmeans" +} \ No newline at end of file diff --git a/resources/integration_tests/infra/aws_sagemaker_notebook_instances.tf b/resources/integration_tests/infra/aws_sagemaker_notebook_instances.tf new file mode 100644 index 000000000..1a30b4306 --- /dev/null +++ b/resources/integration_tests/infra/aws_sagemaker_notebook_instances.tf @@ -0,0 +1,30 @@ +resource "aws_iam_role" "sagemaker_notebook_instance_iam_role" { + name = "sagemaker-notebook-instance-iam-role-${var.test_prefix}${var.test_suffix}" + path = "/" + + assume_role_policy = <