feat(client-glue): Added ability to create data quality rulesets for …

…shared, cross-account Glue Data Catalog tables. Added support for dataset comparison rules through a new parameter called AdditionalDataSources. Enhanced the data quality results with a map containing profiled metric values.
aws · May 25, 2023 · d55340d · d55340d
1 parent 75a8506
commit d55340d
Show file tree

Hide file tree

Showing 20 changed files with 612 additions and 201 deletions.
diff --git a/clients/client-glue/src/commands/BatchGetDataQualityResultCommand.ts b/clients/client-glue/src/commands/BatchGetDataQualityResultCommand.ts
@@ -79,6 +79,9 @@ export interface BatchGetDataQualityResultCommandOutput extends BatchGetDataQual
  * //           Description: "STRING_VALUE",
  * //           EvaluationMessage: "STRING_VALUE",
  * //           Result: "PASS" || "FAIL" || "ERROR",
+ * //           EvaluatedMetrics: { // EvaluatedMetricsMap
+ * //             "<keys>": Number("double"),
+ * //           },
  * //         },
  * //       ],
  * //     },

diff --git a/clients/client-glue/src/commands/BatchGetJobsCommand.ts b/clients/client-glue/src/commands/BatchGetJobsCommand.ts
@@ -1007,6 +1007,28 @@ export interface BatchGetJobsCommandOutput extends BatchGetJobsResponse, __Metad
  * //             },
  * //             Inputs: "<OneInput>",
  * //           },
+ * //           EvaluateDataQualityMultiFrame: { // EvaluateDataQualityMultiFrame
+ * //             Name: "STRING_VALUE", // required
+ * //             Inputs: [ // required
+ * //               "STRING_VALUE",
+ * //             ],
+ * //             AdditionalDataSources: { // DQDLAliases
+ * //               "<keys>": "STRING_VALUE",
+ * //             },
+ * //             Ruleset: "STRING_VALUE", // required
+ * //             PublishingOptions: {
+ * //               EvaluationContext: "STRING_VALUE",
+ * //               ResultsS3Prefix: "STRING_VALUE",
+ * //               CloudWatchMetricsEnabled: true || false,
+ * //               ResultsPublishingEnabled: true || false,
+ * //             },
+ * //             AdditionalOptions: { // DQAdditionalOptions
+ * //               "<keys>": "STRING_VALUE",
+ * //             },
+ * //             StopJobOnFailureOptions: {
+ * //               StopJobOnFailureTiming: "Immediate" || "AfterDataLoad",
+ * //             },
+ * //           },
  * //         },
  * //       },
  * //       ExecutionClass: "FLEX" || "STANDARD",

diff --git a/clients/client-glue/src/commands/CreateDataQualityRulesetCommand.ts b/clients/client-glue/src/commands/CreateDataQualityRulesetCommand.ts
@@ -54,6 +54,7 @@ export interface CreateDataQualityRulesetCommandOutput extends CreateDataQuality
  *   TargetTable: { // DataQualityTargetTable
  *     TableName: "STRING_VALUE", // required
  *     DatabaseName: "STRING_VALUE", // required
+ *     CatalogId: "STRING_VALUE",
  *   },
  *   ClientToken: "STRING_VALUE",
  * };

diff --git a/clients/client-glue/src/commands/CreateJobCommand.ts b/clients/client-glue/src/commands/CreateJobCommand.ts
@@ -998,6 +998,28 @@ export interface CreateJobCommandOutput extends CreateJobResponse, __MetadataBea
  *         },
  *         Inputs: "<OneInput>",
  *       },
+ *       EvaluateDataQualityMultiFrame: { // EvaluateDataQualityMultiFrame
+ *         Name: "STRING_VALUE", // required
+ *         Inputs: [ // required
+ *           "STRING_VALUE",
+ *         ],
+ *         AdditionalDataSources: { // DQDLAliases
+ *           "<keys>": "STRING_VALUE",
+ *         },
+ *         Ruleset: "STRING_VALUE", // required
+ *         PublishingOptions: {
+ *           EvaluationContext: "STRING_VALUE",
+ *           ResultsS3Prefix: "STRING_VALUE",
+ *           CloudWatchMetricsEnabled: true || false,
+ *           ResultsPublishingEnabled: true || false,
+ *         },
+ *         AdditionalOptions: { // DQAdditionalOptions
+ *           "<keys>": "STRING_VALUE",
+ *         },
+ *         StopJobOnFailureOptions: {
+ *           StopJobOnFailureTiming: "Immediate" || "AfterDataLoad",
+ *         },
+ *       },
  *     },
  *   },
  *   ExecutionClass: "FLEX" || "STANDARD",

diff --git a/clients/client-glue/src/commands/CreateMLTransformCommand.ts b/clients/client-glue/src/commands/CreateMLTransformCommand.ts
@@ -14,7 +14,7 @@ import { HttpRequest as __HttpRequest, HttpResponse as __HttpResponse } from "@s
 import { SerdeContext as __SerdeContext } from "@smithy/types";
 
 import { GlueClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from "../GlueClient";
-import { CreateMLTransformRequest, CreateMLTransformResponse } from "../models/models_0";
+import { CreateMLTransformRequest, CreateMLTransformResponse } from "../models/models_1";
 import { de_CreateMLTransformCommand, se_CreateMLTransformCommand } from "../protocols/Aws_json1_1";
 
 /**

diff --git a/clients/client-glue/src/commands/GetDataQualityResultCommand.ts b/clients/client-glue/src/commands/GetDataQualityResultCommand.ts
@@ -75,6 +75,9 @@ export interface GetDataQualityResultCommandOutput extends GetDataQualityResultR
  * //       Description: "STRING_VALUE",
  * //       EvaluationMessage: "STRING_VALUE",
  * //       Result: "PASS" || "FAIL" || "ERROR",
+ * //       EvaluatedMetrics: { // EvaluatedMetricsMap
+ * //         "<keys>": Number("double"),
+ * //       },
  * //     },
  * //   ],
  * // };

diff --git a/clients/client-glue/src/commands/GetDataQualityRulesetCommand.ts b/clients/client-glue/src/commands/GetDataQualityRulesetCommand.ts
@@ -55,6 +55,7 @@ export interface GetDataQualityRulesetCommandOutput extends GetDataQualityRulese
  * //   TargetTable: { // DataQualityTargetTable
  * //     TableName: "STRING_VALUE", // required
  * //     DatabaseName: "STRING_VALUE", // required
+ * //     CatalogId: "STRING_VALUE",
  * //   },
  * //   CreatedOn: new Date("TIMESTAMP"),
  * //   LastModifiedOn: new Date("TIMESTAMP"),

diff --git a/clients/client-glue/src/commands/GetDataQualityRulesetEvaluationRunCommand.ts b/clients/client-glue/src/commands/GetDataQualityRulesetEvaluationRunCommand.ts
@@ -88,6 +88,19 @@ export interface GetDataQualityRulesetEvaluationRunCommandOutput
  * //   ResultIds: [ // DataQualityResultIdList
  * //     "STRING_VALUE",
  * //   ],
+ * //   AdditionalDataSources: { // DataSourceMap
+ * //     "<keys>": {
+ * //       GlueTable: {
+ * //         DatabaseName: "STRING_VALUE", // required
+ * //         TableName: "STRING_VALUE", // required
+ * //         CatalogId: "STRING_VALUE",
+ * //         ConnectionName: "STRING_VALUE",
+ * //         AdditionalOptions: {
+ * //           "<keys>": "STRING_VALUE",
+ * //         },
+ * //       },
+ * //     },
+ * //   },
  * // };
  *
  * ```

diff --git a/clients/client-glue/src/commands/GetJobCommand.ts b/clients/client-glue/src/commands/GetJobCommand.ts
@@ -1003,6 +1003,28 @@ export interface GetJobCommandOutput extends GetJobResponse, __MetadataBearer {}
  * //           },
  * //           Inputs: "<OneInput>",
  * //         },
+ * //         EvaluateDataQualityMultiFrame: { // EvaluateDataQualityMultiFrame
+ * //           Name: "STRING_VALUE", // required
+ * //           Inputs: [ // required
+ * //             "STRING_VALUE",
+ * //           ],
+ * //           AdditionalDataSources: { // DQDLAliases
+ * //             "<keys>": "STRING_VALUE",
+ * //           },
+ * //           Ruleset: "STRING_VALUE", // required
+ * //           PublishingOptions: {
+ * //             EvaluationContext: "STRING_VALUE",
+ * //             ResultsS3Prefix: "STRING_VALUE",
+ * //             CloudWatchMetricsEnabled: true || false,
+ * //             ResultsPublishingEnabled: true || false,
+ * //           },
+ * //           AdditionalOptions: { // DQAdditionalOptions
+ * //             "<keys>": "STRING_VALUE",
+ * //           },
+ * //           StopJobOnFailureOptions: {
+ * //             StopJobOnFailureTiming: "Immediate" || "AfterDataLoad",
+ * //           },
+ * //         },
  * //       },
  * //     },
  * //     ExecutionClass: "FLEX" || "STANDARD",

diff --git a/clients/client-glue/src/commands/GetJobsCommand.ts b/clients/client-glue/src/commands/GetJobsCommand.ts
@@ -1005,6 +1005,28 @@ export interface GetJobsCommandOutput extends GetJobsResponse, __MetadataBearer
  * //             },
  * //             Inputs: "<OneInput>",
  * //           },
+ * //           EvaluateDataQualityMultiFrame: { // EvaluateDataQualityMultiFrame
+ * //             Name: "STRING_VALUE", // required
+ * //             Inputs: [ // required
+ * //               "STRING_VALUE",
+ * //             ],
+ * //             AdditionalDataSources: { // DQDLAliases
+ * //               "<keys>": "STRING_VALUE",
+ * //             },
+ * //             Ruleset: "STRING_VALUE", // required
+ * //             PublishingOptions: {
+ * //               EvaluationContext: "STRING_VALUE",
+ * //               ResultsS3Prefix: "STRING_VALUE",
+ * //               CloudWatchMetricsEnabled: true || false,
+ * //               ResultsPublishingEnabled: true || false,
+ * //             },
+ * //             AdditionalOptions: { // DQAdditionalOptions
+ * //               "<keys>": "STRING_VALUE",
+ * //             },
+ * //             StopJobOnFailureOptions: {
+ * //               StopJobOnFailureTiming: "Immediate" || "AfterDataLoad",
+ * //             },
+ * //           },
  * //         },
  * //       },
  * //       ExecutionClass: "FLEX" || "STANDARD",

diff --git a/clients/client-glue/src/commands/GetUserDefinedFunctionCommand.ts b/clients/client-glue/src/commands/GetUserDefinedFunctionCommand.ts
@@ -14,7 +14,8 @@ import { HttpRequest as __HttpRequest, HttpResponse as __HttpResponse } from "@s
 import { SerdeContext as __SerdeContext } from "@smithy/types";
 
 import { GlueClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from "../GlueClient";
-import { GetUserDefinedFunctionRequest, GetUserDefinedFunctionResponse } from "../models/models_1";
+import { GetUserDefinedFunctionRequest } from "../models/models_1";
+import { GetUserDefinedFunctionResponse } from "../models/models_2";
 import { de_GetUserDefinedFunctionCommand, se_GetUserDefinedFunctionCommand } from "../protocols/Aws_json1_1";
 
 /**

diff --git a/clients/client-glue/src/commands/GetUserDefinedFunctionsCommand.ts b/clients/client-glue/src/commands/GetUserDefinedFunctionsCommand.ts
@@ -14,8 +14,7 @@ import { HttpRequest as __HttpRequest, HttpResponse as __HttpResponse } from "@s
 import { SerdeContext as __SerdeContext } from "@smithy/types";
 
 import { GlueClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from "../GlueClient";
-import { GetUserDefinedFunctionsRequest } from "../models/models_1";
-import { GetUserDefinedFunctionsResponse } from "../models/models_2";
+import { GetUserDefinedFunctionsRequest, GetUserDefinedFunctionsResponse } from "../models/models_2";
 import { de_GetUserDefinedFunctionsCommand, se_GetUserDefinedFunctionsCommand } from "../protocols/Aws_json1_1";
 
 /**

diff --git a/clients/client-glue/src/commands/ListDataQualityRulesetsCommand.ts b/clients/client-glue/src/commands/ListDataQualityRulesetsCommand.ts
@@ -56,6 +56,7 @@ export interface ListDataQualityRulesetsCommandOutput extends ListDataQualityRul
  *     TargetTable: { // DataQualityTargetTable
  *       TableName: "STRING_VALUE", // required
  *       DatabaseName: "STRING_VALUE", // required
+ *       CatalogId: "STRING_VALUE",
  *     },
  *   },
  *   Tags: { // TagsMap
@@ -74,6 +75,7 @@ export interface ListDataQualityRulesetsCommandOutput extends ListDataQualityRul
  * //       TargetTable: { // DataQualityTargetTable
  * //         TableName: "STRING_VALUE", // required
  * //         DatabaseName: "STRING_VALUE", // required
+ * //         CatalogId: "STRING_VALUE",
  * //       },
  * //       RecommendationRunId: "STRING_VALUE",
  * //       RuleCount: Number("int"),

diff --git a/clients/client-glue/src/commands/StartDataQualityRulesetEvaluationRunCommand.ts b/clients/client-glue/src/commands/StartDataQualityRulesetEvaluationRunCommand.ts
@@ -74,6 +74,19 @@ export interface StartDataQualityRulesetEvaluationRunCommandOutput
  *   RulesetNames: [ // RulesetNames // required
  *     "STRING_VALUE",
  *   ],
+ *   AdditionalDataSources: { // DataSourceMap
+ *     "<keys>": {
+ *       GlueTable: {
+ *         DatabaseName: "STRING_VALUE", // required
+ *         TableName: "STRING_VALUE", // required
+ *         CatalogId: "STRING_VALUE",
+ *         ConnectionName: "STRING_VALUE",
+ *         AdditionalOptions: {
+ *           "<keys>": "STRING_VALUE",
+ *         },
+ *       },
+ *     },
+ *   },
  * };
  * const command = new StartDataQualityRulesetEvaluationRunCommand(input);
  * const response = await client.send(command);

diff --git a/clients/client-glue/src/commands/UpdateJobCommand.ts b/clients/client-glue/src/commands/UpdateJobCommand.ts
@@ -995,6 +995,28 @@ export interface UpdateJobCommandOutput extends UpdateJobResponse, __MetadataBea
  *           },
  *           Inputs: "<OneInput>",
  *         },
+ *         EvaluateDataQualityMultiFrame: { // EvaluateDataQualityMultiFrame
+ *           Name: "STRING_VALUE", // required
+ *           Inputs: [ // required
+ *             "STRING_VALUE",
+ *           ],
+ *           AdditionalDataSources: { // DQDLAliases
+ *             "<keys>": "STRING_VALUE",
+ *           },
+ *           Ruleset: "STRING_VALUE", // required
+ *           PublishingOptions: {
+ *             EvaluationContext: "STRING_VALUE",
+ *             ResultsS3Prefix: "STRING_VALUE",
+ *             CloudWatchMetricsEnabled: true || false,
+ *             ResultsPublishingEnabled: true || false,
+ *           },
+ *           AdditionalOptions: { // DQAdditionalOptions
+ *             "<keys>": "STRING_VALUE",
+ *           },
+ *           StopJobOnFailureOptions: {
+ *             StopJobOnFailureTiming: "Immediate" || "AfterDataLoad",
+ *           },
+ *         },
  *       },
  *     },
  *     ExecutionClass: "FLEX" || "STANDARD",